{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的 MNIST 数据函数为我们读入数据，如果没有下载的话则进行下载。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-698ada706af1>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个非常非常简陋的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#包装函数方便调优\n",
    "\n",
    "def train(step = 0.5, layer_nodes = [500,200]):\n",
    "    # Create the model\n",
    "    x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "\n",
    "    # W = tf.Variable(tf.zeros([784, 10]))\n",
    "    # b = tf.Variable(tf.zeros([10]))\n",
    "    # y = tf.matmul(x,W) + b\n",
    "\n",
    "#     如果权重矩阵使用全0来初始化的话，不增加激活函数，会得到很差的结果，大概0.1左右，\n",
    "#     如果增加激活函数，也没有很好的提升,所以使用正太分布来初始化权重矩阵\n",
    "    \n",
    "#     WW = []\n",
    "#     current_node = 784\n",
    "#     for node_count in layer_nodes:\n",
    "#         W = tf.Variable(tf.truncated_normal([current_node, node_count],stddev = 0.1))\n",
    "#         current_node = node_count\n",
    "#         WW.append(W)\n",
    "#     W = tf.Variable(tf.truncated_normal([current_node, 10],stddev = 0.1))\n",
    "#     WW.append(W)\n",
    "\n",
    "    \n",
    "\n",
    "#     W1 = tf.Variable(tf.truncated_normal([784, 500],stddev = 0.1))\n",
    "#     W2 = tf.Variable(tf.truncated_normal([500, 400],stddev = 0.1))\n",
    "#     W3 = tf.Variable(tf.truncated_normal([400, 10],stddev = 0.1))\n",
    "    \n",
    "#     bb = []\n",
    "#     for node_count in layer_nodes:\n",
    "#         _b = tf.Variable(tf.constant(0.1,shape = [node_count]))\n",
    "#         bb.append(_b)\n",
    "#     b = tf.Variable(tf.constant(0.1,shape = [10]))\n",
    "#     bb.append(b)\n",
    "\n",
    "    \n",
    "        \n",
    "#     b1 = tf.Variable(tf.constant(0.1,shape = [500]))\n",
    "#     b2 = tf.Variable(tf.constant(0.1,shape = [400]))\n",
    "#     b3 = tf.Variable(tf.constant(0.1,shape = [10]))\n",
    "    \n",
    "    y = x\n",
    "    current_node = 784\n",
    "    for i in range(len(layer_nodes)):\n",
    "        node_count = layer_nodes[i]\n",
    "        print(str(current_node) + \"_\" + str(node_count))\n",
    "        W = tf.Variable(tf.truncated_normal([current_node, node_count],stddev = 0.1))\n",
    "        b = tf.Variable(tf.constant(0.1,shape = [node_count]))\n",
    "        y = tf.nn.relu(tf.matmul(y, W) + b)\n",
    "        current_node = node_count\n",
    "    W = tf.Variable(tf.truncated_normal([current_node, 10],stddev = 0.1))\n",
    "    b = tf.Variable(tf.constant(0.1,shape = [10]))\n",
    "    y = tf.matmul(y, W) + b\n",
    "    \n",
    "    \n",
    "#     y1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "#     y2 = tf.nn.relu(tf.matmul(y1, W2) + b2)\n",
    "#     y = tf.matmul(y2, W3) + b3\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    # Define loss and optimizer\n",
    "    y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "    # The raw formulation of cross-entropy,\n",
    "    #\n",
    "    #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "    #                                 reduction_indices=[1]))\n",
    "    #\n",
    "    # can be numerically unstable.\n",
    "    #\n",
    "    # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "    # outputs of 'y', and then average across the batch.\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    "    )\n",
    "    train_step = tf.train.GradientDescentOptimizer(step).minimize(cross_entropy)\n",
    "\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    # Train\n",
    "    for _ in range(3000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    ret = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "    print(ret)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "锁定relu，尝试增加隐层，改变隐层节点数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_100\n",
      "0.976\n",
      "784_200\n",
      "0.9785\n",
      "784_300\n",
      "0.9787\n",
      "784_400\n",
      "0.9801\n",
      "784_500\n",
      "0.9779\n",
      "784_600\n",
      "0.9779\n",
      "784_700\n",
      "0.9789\n",
      "784_800\n",
      "0.9817\n",
      "784_900\n",
      "0.9801\n",
      "784_1000\n",
      "0.9816\n",
      "784_1100\n",
      "0.9793\n",
      "784_1200\n",
      "0.9809\n",
      "784_1300\n",
      "0.9809\n",
      "784_1400\n",
      "0.9798\n",
      "784_1500\n",
      "0.9815\n",
      "784_1600\n",
      "0.9811\n",
      "784_1700\n",
      "0.98\n",
      "784_1800\n",
      "0.9809\n",
      "784_1900\n",
      "0.9802\n",
      "784_2000\n",
      "0.9803\n",
      "评分最好的i和acc是：7,0.9817\n"
     ]
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "acc_array = {}\n",
    "\n",
    "best_i = 0\n",
    "best_acc = 0\n",
    "for i in range(20):\n",
    "    node_count = (i+1) * 100\n",
    "    _layer_nodes = [node_count] #[600,500,400,10]\n",
    "    accuraccy = train(step = 0.5, layer_nodes = _layer_nodes)\n",
    "    acc_array[i] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_i = i\n",
    "    \n",
    "print(\"评分最好的i和acc是：\"+ str(best_i) + \",\" + str(best_acc))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])\n",
      "dict_values([0.976, 0.9785, 0.9787, 0.9801, 0.9779, 0.9779, 0.9789, 0.9817, 0.9801, 0.9816, 0.9793, 0.9809, 0.9809, 0.9798, 0.9815, 0.9811, 0.98, 0.9809, 0.9802, 0.9803])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1ce6fdd8>]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYFOW1+PHvYVfBDRAXuICCIgqiM4xeo6K4wcSguCRoYlwSTRSy/lyjyU1MjErUe2Pcd73xqhE1YBwUgqAmbgzKgDPIqgaEICoBUdmc8/vjVIWm6Znpma7uqp4+n+eZZ3q6q6ve7pnpU+973jqvqCrOOedcm7gb4JxzLhk8IDjnnAM8IDjnnAt4QHDOOQd4QHDOORfwgOCccw7wgOCccy7gAcE55xzgAcE551ygXdwNaI5u3bppnz594m6Gc84VlVmzZn2kqt2b2q6oAkKfPn2orq6OuxnOOVdUROT9bLbzISPnnHOABwTnnHMBDwjOOecADwjOOecCHhCcc84BHhCcc84FPCA455wDPCC4IrBoETz7bNytcK7184DgEu83v4FTToG1a+NuiXOtmwcEl3i1tbB5M/z1r3G3xLnWzQOCS7T6epg3z25XVcXbFudaOw8ILtGWLoXPPoMOHWDyZFAtfBtuuQUOPDCeYztXSB4QXKLV1tr3s8+G5cthzpzCt+H++60dS5YU/tjOFZIHBJdodXX2/ac/te+FHjZ67z2oqbHbs2YV9tjOFZoHBJdodXXQowcMHAhlZYUPCJMm2fc2bcArr7fMihXw+9/Dhg1xt8Q1xQOCS7S6OgsGAJWV8MorsHp14Y4/caId/5BDvIfQUuPHw49/DMcfD598EndrXGM8ILjEUrWAcMAB9nNlpc06mjKlMMdfvRpefBFGjYLycgsInlhuvqlToXdveP11OPxwWLw47ha5hnhAcIm1bBl8+umWHsLQodC1a+GGjaqq4Msv4eSTbbhqzRr/MGuuDz6whPzYsTBtGqxaBYcdBq++GnfLXCYeEFxihQnlMCC0bQsnnmjTT+vr83/8SZNg992hosJ6CODDRs0VXkx4wglwxBHw2muw004wfDg8+WS8bXPb8oDgEis9IIANG61alf8P5g0bLPB87WuWUD7gAOjY0RPLzTVlCuy2GwwaZD/372+9g0MOgTPOgBtv9GG4JPGA4BKrrg66d7ev0Ikngkj+h41mzLDhqpNPtp/bt4fBg72H0Bz19ZY/OP54C6qh7t1t+OiMM+DSS204afPm+NrptvCA4BKrtnbr3gFAt25w6KH5DwgTJ8L228Oxx265L0wsF2K4qjWYM8d6c8cfv+1jnTrBo4/C5ZfDHXdY4F23rvBtdFvzgOASKZxhlB4QwIaNZs60D5t8HXvSJOuNdOq05f6yMqu46onl7ISzwTIFBLBew/XXw113wfPPw5FHWhLaxccDgkukFStsVk9DAUHVPkTyYdYs+2AKh4tCnlhunqlTLfey556Nb3fhhfCXv9i6F4cdFk95Emc8ILhEChPK4TUIqQ4+2K5eztew0cSJdvb61a9uff/AgZ5YztYXX8DLL9vsomyMGAF/+5sF+iOOyF+wd43zgOASKSxql6mH0KYNjBwJzz1n1wlEbdIk+1Dq1m3r+9u3hyFDvIeQjZdftplaDQ0XZXLQQTYtde+9LRjfc0/+2ucy84DgEqmuDnbd1aYsZlJZaVcSv/56tMd9910bskgfLgqVlXliORtTpljJ8qOOat7zeva0YHL88TaUdOWV/l4XkgcEl0hhQlkk8+PHH28XqkU9bBQWs2ssIHz6qY13u4ZNnWq9rB12aP5zu3SBZ56B733Pks5nnQXr10ffRrctDwgucVQzTzlNtfPOVhcn6oAQFrPbZ5/Mj4eJZc8jNOyf/7ReVnOGi9K1a2fTUcePh8cft1zEpk3RtdFlllVAEJERIjJfRBaJyBUZHu8tItNEZI6IzBCRnimPjReRWhGZJyK3iNg5n4hcKyJLRcRnH7utrFxpw0GZEsqpKivhrbds4ZwofPIJvPRSw70DsGDRqZPnERqTWq4iFyJ24do999gw0p//nHvbXOOaDAgi0ha4DRgJDATOFJH0c7cbgYdVdTBwDXBd8NzDga8Ag4EDgaHAsOA5zwAVEbwG18pkKlmRSWWlfX/uuWiOm1rMriHt2nliuSlTplhCfsiQaPZ33nnQpw/cems0+3MNy6aHUAEsUtUlqroReAxI/5cZCEwLbk9PeVyBTkAHoCPQHlgJoKqvqeqK3JrvWqNsA8KgQbDXXtENG02aBHvsYVVVG1NWBm++6cnOTFQtf3DccVuXq8hF27Zw8cXWe5s7N5p9usyy+ZXtBSxN+XlZcF+qGuC04PZooIuIdFXVV7EAsSL4el5V5+XWZNfa1dVZRcw99mh8OxHrJUydmvv4cnoxu8aEieWFC3M7Zmv09tuWQ8glf5DJ+efbUN1tt0W7X7e1bAJCpnke6fUJLwGGichb2JDQB8BmEekH7A/0xILIcBFp1kQ0EblQRKpFpHpVvmoVuESprbX8QUMzjFJVVlo5iVdeye2Y06dbLZ3GhotCnlhuWFPlKlqqa1c480z43/+Ff/0r2n27LbIJCMuAXik/9wS2SuOp6nJVPVVVDwauCu5bg/UWXlPVdaq6DpgMHNacBqrq3aparqrl3VPLXrpWq6EaRpkce6xdMJbrsNHEiTZFcvjwprfdf3/YbjvPI2QydSoMGAC9ejW9bXONHQuffw4PPRT9vp1pl8U2M4H+ItIXO/MfA5yVuoGIdAM+UdV64Erg/uChfwAXiMh1WE9jGPA/EbXdtUKrVsFHH2UfELp0sYufqqrghhtadsz6+szF7BoSJpYL1UOYMQP+679yy1n07m0fpG3bRtasbaxfb0uOXnhhfvZfVma1jm6/HX7wg+hyFE2prYWbboL//m8bymzNmnxLVXUzMA54HpgH/ElVa0XkGhEZFWx2NDBfRBYAPYBrg/snAIuBuVieoUZVn4F/T0ddBmwvIstE5JfRvSxXrLJNKKeqrLSx63/8o2XHnDXLpq5mM1wUKiuzKa+FSCzfcosdq0OHln1t3AiPPJL/kuF//7sFhVynmzZm7FhYsGDL1NZC+OEP4YEH4JJLCnfM2Khq0XyVlZWpa91uv10VVJcuzf45dXX2nDvvbNkxr75atW1b1Y8+yv45Dz5ox5w3r2XHzNaGDaqdO6t+73st38fGjap77ql64onRtSuTyy5Tbd9e9dNP83eM9etVu3dXHTUqf8dI9cIL9nseMMC+T5lSmONGDajWLD5j/Upllyi1tTYMtFf6PLZGDBhg89RbegY8caKVWejaNfvnlJXZ93wPG/3975bsDq+5aIn27eH737cKogsWRNe2dFOn2tXjnTvn7xgdO8IFF1hpi/fey99xwKbQ/vzn9rf46quw337w3e/aDLPWygOCS5SmahhlEk4//etfbfpoc7z7rs1tHzWq6W1TDRhQmMRyVZUN+2ST7G7MBRdYYLjjjmjale7DD21YK+rZRZl8//v2O7/zzvweZ8oUC8hXX22lUu6/H5YutVXeWisPCC5RmjPDKFVlpc1Aeeml5j1v4kT73pz8AVhi+eCD899DqKqypHmuZ9277w6nn25j4Z99Fk3bUk0LLkvNZ/4g1KuX/b7uvTd/Re9ULRD06WPXQID1fn70Iwuq06fn57hx84DgEuPjj62OUVM1jDI55hgbTpg8uXnPmzjRjtdQMbvGhInlfKzJADYkUleX23BRqrFjbRW6Rx6JZn+ppkyBXXaBQw6Jft+ZjBtnfy+PP56f/T/zjAX7n//cemiha6+1v5Xvfjc/gTVuHhBcYrRkhlFo++0tKDQnj/DJJ1Y0rbm9g1B5uX0ozJ/fsuc3JQxuUQWEww+36bK33mpnwFFRtYBw3HH5ndaa6phj7HqQfNQ3qq+3QNCvH3z721s/tv32cN99sGQJ/Oxn0R87bh4QXGLkEhDAPjjnz4fFi7Pb/tlnmy5m15gwsZyvPEJVla0etu++0exPxHoJc+facpVRmTfPpu0WYrgoFL6W6mp4441o9/3kk1a++5e/tKHBdMOG2bH/8Ido38ck8IDgEqOuzq4WbulVriNH2vdsh43CYnZhKYrmGjDAzhjzERDWr7dx+crK5iXYm3LWWZYgjfLMOl/lKppy9tmWW4nytXz5pV0EOHAgjBnT8HbXX28X+51/vuWuWgsPCC4x6upsGKClV6D262dn09kMG23YYGWzsylm15C2bfOXWH7xRVuoPqrhotD229uH2FNPRbeOxNSp9r737h3N/rK1445wzjmWR4iqzNmjj1qP51e/anz4q3NnS2ovXGgBpLXwgOASIyxql4vKSpsB0tRZ2wsvZF/MrjH5SixXVVkZjaOPjna/ABddZO29++7c97Vhg5XWKHTvIHTxxXYl9r335r6vTZssEBx0EJx6atPbH3uslem4+ebo1/aOiwcElwirV8OKFS3PH4RGjrThlhkzGt+uOcXsGlNebsHnnXdy20+6yZOtbdttF+1+wXpSI0bAXXfZh2kuXn3VXn8h8wepBg609+nOO2Hz5tz29fDDtlb2r3+dfa/xd7+zC9fOO691rPvsAcElwrxglYxcA8JRR9mwSGPDRmExuxEjsitm15h8JJYXLrSvqIeLUo0bZ+sWPP10bvuZMsWGVvLRk8nW2LFWx+ovf2n5PjZsgGuugYoKOOmk7J+34462xOe8efb8YucBwSVCrjOMQp06WVf+2WcbnlpZXW29kVyHi8DKGeywQ7R5hDApHibJ82HECJvBlOuCM1Onwn/+p30wxmXUKOjZM7fXct99FlR+/evmJ/FPPNF6COPHF39JdA8ILhFqa214pE+f3PdVWWkXdTV0fcCkSXZW+9Wv5n6sMLEc5QdBVZUFmr33jm6f6dq0sfH3l1+2KZYt8fHH9rrjyh+E2rWzchZ//WvLhu6++MIuODviiJa/lptvhh49LDDkOgwXJw8ILhFynWGUqqnpp2Exu113zf1YYMNGs2fnPoYNdqHbjBn5HS4KnXdebstSTptmvbC48gepLrjArii+/fbmP/euu2zG1W9+0/IpvjvvbPuZO9eCS7HygOASoaU1jDLp3dtmK2XKIyxZYmsnRDFcFIoysTx9uo1nFyIg7LorfPOb8Mc/tmxZyilT7IOwpddxRGm33eCMM+DBB5tXjfSzz+C662yYcdiw3Npw0knwrW/Bb39rJwjFyAOCi93atbBsWXQBAewD9cUXbWppqpYWs2tMlInlqirLSRx5ZO77yka4LOWDDzbveWG5iuHDM1/NG4dx4ywY/PGP2T/n1lutUuuvfx1NG37/eyujft55No212HhAcLELZxjleg1CqspK+4cMq3CGJk6EAw+Mdnx+333tQqVcE8uqFhCOO84K9RXCwQdbjaPbbmve6m8LFlgp6CQMF4UOPdSK62Vbq2ntWksEV1ZaYjwKu+5q1VBnz275kq5x8oDgYldba9+j7CF85Su20E7qsNHHH+dWzK4hUSWW582D998vzHBRqrFjbf791KnZPyeuchWNEbFeQl2d9Q6b8j//YwUOo54uOno0fOMbtt+3345mn4VKVHtAcLGrq7Mz4r59o9tn+/Z29lpVteVs8dln7Sw46oAA0SSWw+CVz+mmmZx+us2QaU5NoKlTrQx0PmdCtcSYMXaW3tRr+eQTuOkm+/AOh/yi9Ic/WH7l/POb/zehags3Pfoo/PjHcNhhdnKzbFn07UznAcHFrq7OCsVFXTp55Ej7JwrP0sJidvn4ACgvt+mL4fBXS0yeDIMGtby4X0t16GCzdJ591j6ImrJpkyW/k9Q7CG23HXznO/DnPzf+AXrTTZZv+NWv8tOO7t0tKM2caVNSG7NmjU2ZvfZaq63Vo4cF2rPOsvIiHTvawjxRFjlsUDYLLyflq6ysLNqVp10i9O6tetZZ0e/3gw9sYfTrr1f94gvVHXbIbbH6xsybZ8d64IGWPX/NGlug/rLLIm1W1pYuVW3bVvXSS5ve9qWX7LU+9VT+29USS5aoiqhefXXmxz/80P4WvvGN/Lajvl519GjVjh3t70NVdfNm1Zoa1bvuUj3/fNWBA62t1i9QHTBA9dxzVe+4Q/Wtt1Q3bYqmLUC1ZvEZm5D5Aa5UrVtn4+YXXBD9vvfc0xaEqaqyM+/PPsvPcBFsnVg+99zmP3/aNDvzLnT+INSzpw2f3HefnTU3VkMpLFdxzDGFa19z9O1rFx3efbctg5meoL/hBuvN/fKX+W2HiF0XccABViyvRw/rMYQrrXXtaonwMWPse0WFDTPFyYeMXKyiqmHUkMpKWyj94YftAzvXYnYNadPGZri0NLFcVWXlHw4/PNp2NcfYsTa2/thjjW83dWoyPrwaM3asTSd98smt71+xwmZUfetbNkyZb7vvbrOO3n/fAsH559u02IULrWT3s8/a6mwnnJCM99MDgotVVDWMGlJZaaWeH3/c6vfkczpneXnLEsvhdNMTTrBkeFyGDbOz2camba5ebWe5SZpumskJJ1hV1/SrsK+7zn4/v/hF4dry9a9bMHjjDbjlFrsYsF+/AuUEmskDgotVXZ0lNVuyyH02Dj3UFn+H/A0XhcrKrARyGOSyNWeOlU6Ia7goFC5L+eabDdf3f+EFm6mVxIRyqrBW0yuv2HoVYMXr7rrLLhrL199bsfOAUAQuvzyaQmxJVFtrhdzydbVru3ZWjbJt2/x/4Iazl5p7gVo43XTEiGjb0xJnn21DVw1N25wyxR6vqChsu1ri3HOtFHrYSwhrDF19dWxNSrysAoKIjBCR+SKySESuyPB4bxGZJiJzRGSGiPRMeWy8iNSKyDwRuUXEOkoiUiYic4N9/vt+t7VNm6ze+nPPbUlGtSZR1jBqyHXXwTPPRFfMriH9+9t88ebmEaqqLP+wxx75aVdzdO5sy1I+8QSsXLn1Y2G5imOOiXdoK1u77GLDM488YkH6/vtthbP/+I+4W5ZcTQYEEWkL3AaMBAYCZ4pI+r/wjcDDqjoYuAa4Lnju4cBXgMHAgcBQICwhdQdwIdA/+ErA+VHyvPCCjdvW11tXvjX57DMrU53vgNCnT2Eu9mpJYnn1ahvWiHu4KFVDy1IuXmy/r6TnD1KNHWvDeCNHWm/xZz+Lu0XJlk0PoQJYpKpLVHUj8BiQPho7EAirxkxPeVyBTkAHoCPQHlgpInsAO6rqq8Ec2YeBU3J6Ja3UhAlbpgC+8Ua8bYna/Pl21pnvgFBIYWI528JmU6dasE9SQBgwwOoppS9LmcRyFU056CArdf7RRxYcktALS7JsAsJewNKUn5cF96WqAU4Lbo8GuohIV1V9FQsQK4Kv51V1XvD81OsIM+0TABG5UESqRaR61apVWTS39di0yZY4HD3aurkzZ8bdomiFNYyiLGoXt7IyK1+dbWK5qsqGspI2Jj9unF3pO2nSlvumTrXeVr9+sTWrRa66yv7GLrss7pYkXzYBIdPYfvqktEuAYSLyFjYk9AGwWUT6AfsDPbEP/OEiclSW+7Q7Ve9W1XJVLe/evXsWzW09XnzRCrKdcYZ9YLS2HkJdnXXji+0DpjHNSSzX11u5ijDpnSQnnWQnIWFCdvNmG7484YRkTpdszIgRVr5kt93ibknyZRMQlgGp1VV6AstTN1DV5ap6qqoeDFwV3LcG6y28pqrrVHUdMBk4LNhnz8b26Syx17mzfWBUVFidmdbUSaqrsyt8iyFBma1+/WwWTjZ5hDfftIunkjRcFGrbFi66yIJAXZ2djKxdW1zDRa75sgkIM4H+ItJXRDoAY4BJqRuISDcRCfd1JXB/cPsfWM+hnYi0x3oP81R1BfCpiBwWzC76NjAxgtfTamzebMNFJ51kOYShQ+3+1jRsVIgZRoUWJpaz6SFUVdnZ9okn5r9dLfGd72xZlnLKFHtt+brS2yVDkwFBVTcD44DngXnAn1S1VkSuEZFRwWZHA/NFZAHQAwhXFZ0ALAbmYnmGGlV9JnjsIuBeYFGwTQMr4Jaml16y3sDpp9vPZWX24dFaAsIXX9hylq0pfxAqL7eLzZpKLFdVWc8vqSOh3btbnZ2HHrLqoeXl+Z+66+KV1eVAqloFVKXd94uU2xOwD//0530JfK+BfVZjU1FdBhMm2EU14XTJLl3sbLq15BHmz7cx9NbWQ4AtieXaWiuul8mqVfa7zHeBtVyNHWt1oGpq/IKuUuBXKifQl1/CU0/Z1cnbb7/l/jCxnM3ygEmX7xpGccomsfz88/Z7TGL+IFVFxZbhSs8ftH4eEBLob3+zq0TD4aLQ0KE2n/q992JpVqTq6ixx2b9/3C2J3j77wE47NZ5YrqqyWS+HHFK4drXUr34Fxx5rK3e51s0DQgKFF6Olnz2Gc9VbQx6hrs6CQaEWky+kphLLX35pPYSRI23bpBs50lb06tAh7pa4fCuCP8fSUl9vNdxHjrQpp6kGDbIP0NaQR6itbZ3DRaEwsZxpcfQ33rB1B5I+XORKjweEhHnlFVvE44wztn2sQwc4+ODiDwgbNsCiRa07IJSVWTAI13NOVVVlPQMfk3dJ4wEhYZ54wnoBDZW7HjrUxqabuwhLkixY0HpnGIXKy+17pjxCVZWtjBau0+BcUnhASJDU4aIuXTJvU1EBn3/e/EVYkiRse2u8BiG09962JGJ6QFixwq5Q9uEil0QeEBLktdfggw+2nV2UqjUklmtrbchk333jbkn+iGROLD/3nH33gOCSyANCgkyYYHmCr32t4W369bMzz2LOI9TV2dTMTp3ibkl+hYnlDRu23FdVBXvuCYMHx9cu5xriASEh6ustIJx4ohVHa0ibNvZBU+wBoTXnD0JlZVa+Ikwsb9pkNYEqK4uvYqgrDR4QEmLmTFi6tPHholBFBcyda7mEYrNxIyxcWBoBIT2x/MorVjHUh4tcUnlASIgJE6wM9KhRTW9bUWEXN82enf92RW3hQpsh1ZoTyqG+fW0mUZhHqKqy3/Gxx8bbLuca4gEhAVRtuunxx1t+oClhYrkYh41acw2jdGFiOewhTJ4MRx7Z+JCgc3HygJAAs2bB++9nvhgtkz32gJ49izcgiMB++8XdksIoL7fhvUWL7HtYvda5JPKAkABPPGFLSWYzXBQaOrR4A0LfvltXcW3NwsTy+PH2s+cPXJJ5QIiZquUPjjuueYuPVFTA4sVWE6eY1NWVRv4gFCaWH3wQeveG/fePtTnONcoDQszeestWDstmdlGqYrxAbdMmWxinFPIHoT59LLG8aZNPN3XJ5wEhZhMm2LoAp5zSvOeFS2oW07DR4sX2wVhKAUFkSy/Bh4tc0nlAiFE4u2j4cOjatXnP3WknS8wWU0AopRlGqcKZRcccE3dLnGucB4QYzZljs0+ynV2UrqLChoyKZUnNMCCU2jj65ZfbUNkOO8TdEuca5wEhRk88YaUomjtcFKqosKU2ly6Ntl35UltrY+ql9sHYoQPsvnvcrXCuaR4QYhIOFx19NHTv3rJ9FNsFaqVSw8i5YuUBISa1tbZQTEuHi8AqZrZvXxwBYfPm0pth5Fyx8YAQk3C4aPTolu+jY0cYMqQ4pp6++66VgS6laxCcKzYeEGIyYQIcdRT06JHbfioqrHjal19G0658qa21795DcC65sgoIIjJCROaLyCIRuSLD471FZJqIzBGRGSLSM7j/GBGZnfK1XkROCR4bLiJvisjbIvKQiLSL9qUlV12dfTX3YrRMKipg3Tp4553c95VPpTrDyLli0mRAEJG2wG3ASGAgcKaIpJ/n3Qg8rKqDgWuA6wBUdbqqDlHVIcBw4HNgioi0AR4CxqjqgcD7wDkRvabEmzDBLlg69dTc9zV0qH1Peh6hrg569Wp4rWjnXPyy6SFUAItUdYmqbgQeA05O22YgMC24PT3D4wCnA5NV9XOgK7BBVRcEj00FTmtu44vVE0/AEUdY1dJc7beffcgmPY9QajWMnCtG2QSEvYDUme7LgvtS1bDlA3000EVE0q+9HQM8Gtz+CGgvIsFF/ZwO9Mq20cXsnXdsScVcZhelatMm+ZVPv/wS5s3z/IFzSZdNQMhUjiv92thLgGEi8hYwDPgA2PzvHYjsAQwCngdQVcUCxH+LyBvAp6nbb3VwkQtFpFpEqletWpVFc5NtwgT7HsVwUaiiAmpqYP366PYZpffes7Z5QHAu2bIJCMvY+uy9J7A8dQNVXa6qp6rqwcBVwX1rUjb5OvC0qm5Kec6rqnqkqlYALwELMx1cVe9W1XJVLe/e0iu4EmTCBPjKV2Cv9D5WDoYOtXn+SV1Ss1RrGDlXbLIJCDOB/iLSV0Q6YGf2k1I3EJFuQaIY4Erg/rR9nMmW4aLwObsF3zsClwN3Nr/5xWXhQjuTj2J2Uaqkl8L2GUbOFYcmA4KqbgbGYcM984A/qWqtiFwjIuEaX0cD80VkAdADuDZ8voj0wXoYL6bt+lIRmQfMAZ5R1RdyeynJFw4XnRZx+nyvvSxBndQ8wnPPwd57Z7detHMuPqLFUioTKC8v1+rq6rib0WJlZVbo7NVXo9/3KadY4nb+/Oj3nYuZM60Hc+ON8P/+X9ytca40icgsVS1vaju/UrlAliyBN9+MfrgoNHSo1Ub617/ys/+W+t3vbO2GCy6IuyXOuaZ4QCiQcLgoXwEhzCMkqQO1aBE8+SRcdJEtEOOcSzYPCAXyxBN2Ft+7d372Hy7TmKQ8wk03Qbt28MMfxt0S51w2PCAUwHvv2Zl7VBejZbLLLrDvvskJCCtXwgMPwDnnRHNFtnMu/zwg5Fl9Pdxzj93O13BRqKIiOQHh1lth40ZPJDtXTDwg5Mmnn9qH4v77w29/C8ceC3375veYQ4fCihXwwQf5PU5T1q2D226zmU/77RdvW5xz2fOAELElS+CnP4WePeEHP7C59488AlVV+T92UpbUvPdeWL0aLrss3nY455rHA0IEVGH6dDsj7tcP/vAH+OpX7XqD11+Hs86y6w/ybcgQS+LGGRA2bYKbb7bFfw47LL52OOear2QWpcmHL76ws/9bboG5c6FbN/jZz2yaZZS1irLVqRMcdFC8AeHxx2HpUrjjjvja4JxrGQ8ILbBsGdx+O9x9N3zgfhgeAAARL0lEQVT8sS12f999cOaZsN128bZt6FD4v/+zZHabAvf/VGH8eFv3YOTIwh7bOZc7HzLKkqoNAY0ZA336wA032LDIjBlWZfT88+MPBmB5hLVr7arlQnvuOespXXpp4YORcy533kPIwhtvwLhxVpdn553hJz+BsWMtMCRNamJ5wIDCHnv8eEumn3lmYY/rnIuGn8dl4dJL4f33bZho6VKrz5PEYAAWBDp3Lnwe4Y03rLf0k58UJoHunIue9xCaoGprGJx1liWLk65tW6uqWui1EcaP9yJ2zhU77yE04f33Yc0am71TLCoqLK+xYUNhjrdwITz1FFx8MXTpUphjOuei5wGhCeGylMUWEDZuhDlzCnO8m26yYSIvYudccfOA0ISaGhCBQYPibkn2CnnF8sqV8OCDVsRu993zfzznXP54QGhCTQ307w877BB3S7LXqxfstlth8gi33OJF7JxrLTwgNKGmpriGi8B6NIWofPrppzbz6tRTrfS2c664eUBoxNq1VqxuyJC4W9J8FRXwzjuWEM+Xe++1JTsvvTR/x3DOFY4HhEaESdli6yGABQRVmDUrP/sPi9gNGwaHHpqfYzjnCssDQiNqaux7MQaEcEnNfOURHn3Uajp5iWvnWg8PCI2YPRt23TWeyqW56toV9tknP3mEsIjdgQd6ETvnWhO/UrkRNTWWPxCJuyUtU1EBL78c/X4nT4baWnj44eJ9b5xz2/IeQgM2b7bKncU4XBSqqLBhneXLo93v+PE2tXXMmGj365yLlweEBixcCOvXF39AgGjzCK+/Di++aEXs2rePbr/OufhlFRBEZISIzBeRRSJyRYbHe4vINBGZIyIzRKRncP8xIjI75Wu9iJwSPHasiLwZ3P83EekX7UvLTZhQLsYpp6EhQ6zYXZQBYfx42GUXL2LnXGvUZEAQkbbAbcBIYCBwpogMTNvsRuBhVR0MXANcB6Cq01V1iKoOAYYDnwNTgufcAXwzeOz/gKsjeD2RqamxM+D994+7JS23/fZWciOqxPKCBfD001bErnPnaPbpnEuObHoIFcAiVV2iqhuBx4CT07YZCEwLbk/P8DjA6cBkVf08+FmBHYPbOwERj3TnpqbGgkGx1/avqLAeQn197vsKi9j94Ae578s5lzzZBIS9gKUpPy8L7ktVA5wW3B4NdBGRrmnbjAEeTfn5u0CViCwDzgauz7bRhTB7dnEPF4UqKuxq4kWLctvPP/8JDz0E554LPXpE0jTnXMJkExAyTSzUtJ8vAYaJyFvAMOADYPO/dyCyBzAIeD7lOT8BKlW1J/AAcHPGg4tcKCLVIlK9atWqLJqbu1WrYMWK4k4oh4YOte+55hHCInaXXJJ7m5xzyZRNQFgG9Er5uSdpwzuqulxVT1XVg4GrgvtSq+h8HXhaVTcBiEh34CBVfT14/HHg8EwHV9W7VbVcVcu7d++ezWvKWTFfoZxu4EDLJeSSRwiL2J12GvRLVOrfORelbC5Mmwn0F5G+2Jn/GOCs1A1EpBvwiarWA1cC96ft48zg/tBqYCcR2VdVFwDHA/Na9hKiV4yL4jSkXTtbUvORR2BeC9/hjz6yInlepsK51q3JgKCqm0VkHDbc0xa4X1VrReQaoFpVJwFHA9eJiAIvAWPD54tIH6yH8WLaPi8AnhSReixAnB/Vi8pVTY2Vq+jWLe6WROPii23IZ926lj2/Uyf40Y+2DD8551onUU1PByRXeXm5VldX5/04gwfblbjPPpv3QznnXN6JyCxVLW9qO79SOc2GDTa00hqGi5xzrjk8IKSpq7M6Rq1hyqlzzjWHB4Q0rWmGkXPONYcHhDQ1NbDddj690jlXejwgpJk925LKbdvG3RLnnCssDwgpVK2H4MNFzrlS5AEhxbJlsHq1BwTnXGnygJCiNV2h7JxzzeUBIUU4w2jw4Hjb4ZxzcfCAkKKmBvbZB7p0ibslzjlXeB4QUsye7cNFzrnS5QEhsG4dLF7sVyg750qXB4TA3Lk27dR7CM65UuUBIeAzjJxzpc4DQqCmBnbeGf7jP+JuiXPOxcMDQiC8QlkyrSDtnHMlwAMC8OWXlkPw4SLnXCnzgIDNLvrsMw8IzrnS5gGBLVco+5RT51wp84CABYS2bWHgwLhb4pxz8fGAgE05HTAAOnWKuyXOORcfDwhYD8GHi5xzpa7kA8LHH9s6CJ5Qds6VupIPCGFC2QOCc67UeUDwgOCcc4AHBGpqYPfdoUePuFvinHPxyiogiMgIEZkvIotE5IoMj/cWkWkiMkdEZohIz+D+Y0RkdsrXehE5JXjs5ZT7l4vIn6N9adkJS1Y451ypazIgiEhb4DZgJDAQOFNE0mfs3wg8rKqDgWuA6wBUdbqqDlHVIcBw4HNgSvDYkSmPvQo8FdFrytrGjVBb6wHBOecgux5CBbBIVZeo6kbgMeDktG0GAtOC29MzPA5wOjBZVT9PvVNEumDBouA9hHfegU2bfMqpc85BdgFhL2Bpys/LgvtS1QCnBbdHA11EpGvaNmOARzPsfzQwTVXXZjq4iFwoItUiUr1q1aosmps9Tyg759wW2QSETAWhNe3nS4BhIvIWMAz4ANj87x2I7AEMAp7PsK8zyRwo7ECqd6tquaqWd+/ePYvmZm/2bOjYEfbdN9LdOudcUWqXxTbLgF4pP/cElqduoKrLgVMBRKQzcJqqrknZ5OvA06q6KfV5QS+iAuslFFxNDQwaBO2yeRecc66Vy6aHMBPoLyJ9RaQDNvQzKXUDEekmIuG+rgTuT9tHQ72AM4C/qOr65jU7d6o+w8g551I1GRBUdTMwDhvumQf8SVVrReQaERkVbHY0MF9EFgA9gGvD54tIH6yH8WKG3TeUV8i75cvho488IDjnXCirwRJVrQKq0u77RcrtCcCEBp77HtsmocPHjs6ynZHzNRCcc25rJXulchgQBg+Otx3OOZcUJR0Q+vSBnXaKuyXOOZcMJRsQZs/2/IFzzqUqyYDw+eewcKHnD5xzLlVJBoS334b6eu8hOOdcqpIMCLNn23cPCM45t0VJBoSaGthxR0sqO+ecMyUbEAYPhjYl+eqdcy6zkvtIrK/3khXOOZdJyQWEd9+Fdet8hpFzzqUruYDgayA451xmJRkQ2rSBAw+MuyXOOZcsJRcQZs+2BXG22y7uljjnXLKUXECoqfH8gXPOZVJSAeFf/4L33/f8gXPOZVJSAcETys4517CSDAg+ZOScc9squYDQvTvsvnvcLXHOueQpqYAQroEgEndLnHMueUomIGzeDLW1nj9wzrmGlExAmD8fNmzw/IFzzjWkZAKCr4HgnHONK5mAUFMDHTrAgAFxt8Q555KppALCAQdA+/Zxt8Q555KppAKCDxc551zDsgoIIjJCROaLyCIRuSLD471FZJqIzBGRGSLSM7j/GBGZnfK1XkROCR4TEblWRBaIyDwR+WG0L22Lf/4TVq70gOCcc41p19QGItIWuA04HlgGzBSRSapal7LZjcDDqvqQiAwHrgPOVtXpwJBgP7sCi4ApwXPOBXoBA1S1XkR2i+g1bcOvUHbOuaZl00OoABap6hJV3Qg8Bpycts1AYFpwe3qGxwFOByar6ufBzxcB16hqPYCqftjcxmfLaxg551zTsgkIewFLU35eFtyXqgY4Lbg9GugiIl3TthkDPJry8z7AN0SkWkQmi0j/7JvdPLNnQ69esMsu+TqCc84Vv2wCQqZCD5r28yXAMBF5CxgGfABs/vcORPYABgHPpzynI7BeVcuBe4D7Mx5c5MIgaFSvWrUqi+Zua9Ag+OY3W/RU55wrGU3mELAeQa+Un3sCy1M3UNXlwKkAItIZOE1V16Rs8nXgaVXdlLbfJ4PbTwMPZDq4qt4N3A1QXl6eHoiycuWVLXmWc86Vlmx6CDOB/iLSV0Q6YEM/k1I3EJFuIhLu60q2Pds/k62HiwD+DAwPbg8DFjSn4c4556LVZEBQ1c3AOGy4Zx7wJ1WtFZFrRGRUsNnRwHwRWQD0AK4Nny8ifbAexotpu74eOE1E5mKzkr6b0ytxzjmXE1Ft0ShMLMrLy7W6ujruZjjnXFERkVlBvrZRJXOlsnPOucZ5QHDOOQd4QHDOORfwgOCccw7wgOCccy5QVLOMRGQV8H4Ln94N+CjC5kTN25cbb19uvH25SXr7eqtq96Y2KqqAkAsRqc5m2lVcvH258fblxtuXm6S3L1s+ZOSccw7wgOCccy5QSgHh7rgb0ARvX268fbnx9uUm6e3LSsnkEJxzzjWulHoIzjnnGtHqAoKIjBCR+SKySESuyPB4RxF5PHj89aAaa6Ha1ktEpovIPBGpFZEfZdjmaBFZIyKzg69fFKp9wfHfE5G5wbG3qSQo5pbg/ZsjIocUsG37pbwvs0VkrYj8OG2bgr5/InK/iHwoIm+n3LeriEwVkYXB94xr9YnIOcE2C0XknAK273ci8k7w+3taRHZu4LmN/i3ksX2/FJEPUn6HlQ08t9H/9Ty27/GUtr0nIrMbeG7e37/IqWqr+QLaAouBvYEO2NKeA9O2uRi4M7g9Bni8gO3bAzgkuN0FWwMivX1HA3+J8T18D+jWyOOVwGRsJb3DgNdj/F3/E5tfHdv7BxwFHAK8nXLfeOCK4PYVwA0ZnrcrsCT4vktwe5cCte8EoF1w+4ZM7cvmbyGP7fslcEkWv/9G/9fz1b60x28CfhHX+xf1V2vrIVQAi1R1iapuBB4DTk7b5mTgoeD2BOBYEcm0TGjkVHWFqr4Z3P4UW18ifX3qpDsZeFjNa8DOwRKphXYssFhVW3qhYiRU9SXgk7S7U//GHgJOyfDUE4GpqvqJqq4GpgIjCtE+VZ2its4JwGvYKoixaOD9y0Y2/+s5a6x9wefG19l28a+i1doCwl7A0pSfl7HtB+6/twn+KdYAXQvSuhTBUNXBwOsZHv5PEakRkckickBBG2brZU8RkVkicmGGx7N5jwthDA3/I8b5/gH0UNUVYCcBwG4ZtknK+3g+1uPLpKm/hXwaFwxp3d/AkFsS3r8jgZWqurCBx+N8/1qktQWETGf66dOostkmr8TWnX4S+LGqrk17+E1sGOQg4A/YUqOF9BVVPQQYCYwVkaPSHk/C+9cBGAU8keHhuN+/bCXhfbwK2Aw80sAmTf0t5MsdwD7AEGAFNiyTLvb3j8xLA6eK6/1rsdYWEJZhy3WGegLLG9pGRNoBO9GyLmuLiEh7LBg8oqpPpT+uqmtVdV1wuwpoLyLdCtU+VV0efP8QeBrrmqfK5j3Ot5HAm6q6Mv2BuN+/wMpwGC34/mGGbWJ9H4Mk9knANzUY8E6Xxd9CXqjqSlX9UlXrgXsaOG7c71874FTg8Ya2iev9y0VrCwgzgf4i0jc4ixwDTErbZhIQzug4HXihoX+IqAVjjvcB81T15ga22T3MaYhIBfY7+rhA7dtBRLqEt7Hk49tpm00Cvh3MNjoMWBMOjxRQg2dmcb5/KVL/xs4BJmbY5nngBBHZJRgSOSG4L+9EZARwOTBKVT9vYJts/hby1b7UnNToBo6bzf96Ph0HvKOqyzI9GOf7l5O4s9pRf2GzYBZgMxCuCu67BvvjB+iEDTUsAt4A9i5g247AurVzgNnBVyXwfeD7wTbjgFps1sRrwOEFbN/ewXFrgjaE719q+wS4LXh/5wLlBf79bo99wO+Ucl9s7x8WmFYAm7Cz1u9gOalpwMLg+67BtuXAvSnPPT/4O1wEnFfA9i3Cxt/Dv8Fw1t2eQFVjfwsFat//Bn9bc7AP+T3S2xf8vM3/eiHaF9z/YPg3l7Jtwd+/qL/8SmXnnHNA6xsycs4510IeEJxzzgEeEJxzzgU8IDjnnAM8IDjnnAt4QHDOOQd4QHDOORfwgOCccw6A/w+OjI5g/Myn2QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(acc_array.keys())\n",
    "print(acc_array.values())\n",
    "pyplot.plot(acc_array.keys(), acc_array.values(), 'b-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到节点的数量不是越多越好，但是最开始的时候有阶段性提升， 设定首层节点数目为800"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "尝试添加第二层隐层，观察隐层节点数目的规律"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_110\n",
      "0.9811\n",
      "784_800\n",
      "800_120\n",
      "0.98\n",
      "784_800\n",
      "800_130\n",
      "0.9796\n",
      "784_800\n",
      "800_150\n",
      "0.9792\n",
      "784_800\n",
      "800_150\n",
      "0.98\n",
      "784_800\n",
      "800_160\n",
      "0.9815\n",
      "784_800\n",
      "800_170\n",
      "0.9774\n",
      "784_800\n",
      "800_180\n",
      "0.9791\n",
      "784_800\n",
      "800_200\n",
      "0.9791\n",
      "评分最好的i和acc是：160,0.9815\n"
     ]
    }
   ],
   "source": [
    "acc_array = {}\n",
    "\n",
    "best_j = 0\n",
    "best_acc = 0\n",
    "nodes = [110,120,130,150,150,160,170,180,200]\n",
    "for j in range(len(nodes)):\n",
    "    node_count = nodes[j]\n",
    "    _layer_nodes = [800,node_count] #[600,500,400,10]\n",
    "    accuraccy = train(step = 0.5, layer_nodes = _layer_nodes)\n",
    "    acc_array[j] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_j = j\n",
    "    \n",
    "print(\"评分最好的i和acc是：\"+ str(nodes[best_j]) + \",\" + str(best_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "隐层2的节点数目对于准确率的影响\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20f51a58>]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XeYlOXVx/HvoSPWCKIBIxAgCgSZde2JNUbFRCNGXzCWWImvGiuWYCwkFkqQqCgxiIIasQVLImJeS9REE1cFUZEiqDQVNRYUpZ33jzMblnWB2d2Zeab8PtfFxe7sM8+c1V1+8zz3fZ/b3B0REZEmSRcgIiKFQYEgIiKAAkFERNIUCCIiAigQREQkTYEgIiKAAkFERNIUCCIiAigQREQkrVnSBdRH27ZtvVOnTkmXISJSVF588cUP3L3dho4rqkDo1KkTVVVVSZchIlJUzOztTI7TLSMREQEUCCIikqZAEBERQIEgIiJpCgQREQEUCCIikqZAEBERQIEgUrIWLIC77kq6CikmCgSREnXKKXD00fD++0lXIsVCgSBSgp5+Gh59ND5++eVka5HioUAQKTHucPHFsNVW8bkCQTKlQBApMY88Av/8JwwZAp06KRAkcwoEkRKyejUMHgzf/jaceCKkUvDSS0lXJcVCgSBSQu6+G6ZNi6uD5s2hogLmzIFPP026MikGCgSRErFiBVx6KfTuDf37x2OpVPw9bVpydUnxUCCIlIhbb42rgSuvhCbp3+zqQNA4gmRCgSBSApYtgyuugN13h0MOWfP4NttA+/YaR5DMFNWOaSJStxtvhEWL4E9/ArM1j5vFVYKuECQTukIQKXKffgpXXw0HHgh77/31r6dS8Prr8NVX+a9NiosCQaTI/e538OGHMXZQl1QKVq6EV1/Nb11SfBQIIkVsyRIYORJ++lPYaae6j6moiL81jiAbokAQKWJXXw1ffAG/+c26j+ncGTbdVOMIsmEKBJEiNX9+DCYffzxsv/26j2vSBPr0USDIhikQRIrUkCHRyO7yyzd8bCoVi9NWrcp5WVLEFAgiRWjWrFiIdtpp8K1vbfj4iopYqzBzZu5rk+KlQBApQr/+NbRqBb/6VWbHa8WyZKIsAmHxYs2wkNLx8stwzz1wzjlr9jzYkO23h5YtFQiyfiW/Utk9puTNnx/3ULfYIumKRBpn8OD4OT7vvMyf07x5NL3TGyNZn5K/QjCDUaPiKmHgwAgIkWL1zDMweTJcdBFsvnn9nlvdwkK/A7IuGQWCmR1kZjPNbI6ZXVTH17czs8fN7BUze8rMOtb42jAze83MZpjZdWbRacXMrjSz+Wa2NHvfTt123jnmad97L9x2W65fTSQ33GPMYJtt4Iwz6v/8VAo+/hjefjv7tUlp2GAgmFlTYDRwMNADGGBmPWodNgKY4O69gSHA1enn7gHsCfQGegE7A9XdVh4GdsnC95CRQYNg333hzDNh9ux8vapI9kyeDM8+GwPKG21U/+drYFk2JJMrhF2AOe4+192XAxOBw2od0wN4PP3xkzW+7kAroAXQEmgOvAfg7s+7++LGlZ+5pk1hwgRo0QKOPhqWL8/XK4s03urVcXXQpQucdFLDztG7d/weaBxB1iWTQOgAzK/x+YL0YzVNA45If3w4sImZbenuzxEBsTj9Z4q7z6hPgWZ2qplVmVnVkiVL6vPUr+nYEcaOhaqq2FlKpFjce++arTFbtGjYOVq3jtlGukKQdckkEKyOx2oPS50P7G1mLxO3hBYCK82sK7AD0JEIkf3MbK/6FOjuN7t7pbtXtmvXrj5PrVO/fnDqqTBsGDzxRKNPJ5JzK1bEbaJevdZsjdlQ2htB1ieTQFgAbFvj847AopoHuPsid+/n7ilgcPqxT4irhefdfam7LwUmA7tlpfJGGDkSuneHY4+NtsEihey222Lc68or45ZPY6RSsZHOe+9lpTQpMZkEwgtANzPrbGYtgP7AQzUPMLO2ZlZ9rouBcemP3yGuHJqZWXPi6qFet4xyoU0buOuuaB188smahieF68svY2vM3XaDH/+48eerboWtqwSpywYDwd1XAmcAU4h/zO9x99fMbIiZHZo+bB9gppnNAtoD1Vt13Ae8CUwnxhmmufvD8N/pqAuAjcxsgZldnr1va8NSKbjmGnjgAbj55ny+skjmbrwRFi6Eq65ae2vMhurTJ/5WIEhdzIvo7XFlZaVXVVVl7XyrV8PBB8dinxdfhB12yNqpRRrt009jVlFFBTz2WPbO26ULVFZG+wspD2b2ortXbui4kl+pvD5NmsT92TZtYMAA7TkrheXaa2OM66qrsnveVEpTT6VuZR0IEKs+b701pvRdfHHS1YiEDz6IvZL79Yt389lUUQFvvgmffJLd80rxK/tAAPjRj+D00+Md2ZQpSVcjEltjfv45/Pa32T939YrladOyf24pbgqEtOHDoWfP2I7w/feTrkbK2YIFMHo0HHdcbsa11MJC1kWBkNa6dUxF/fhjOOEETUWV5AwZEhMeLrssN+ffZhvYemuNI8jXKRBq+O5340rhkUfiHZpIvs2aBePGwS9+AZ065e51tGJZ6qJAqOWMM6BvXzj/fJg+PelqpNxcdlnsbDZ4cG5fJ5WC11+PhW8i1RQItZjFrKPNN4+uqMuWJV2RlIupU2HiRDj7bGjfPrevlUrBqlV60yNrUyDUYautYPx4ePVVuOCCpKuRclG9NeagQbl/LbWwkLooENbhwANjE/MbboC//CXpaqTUPftsjF1deGH9t8ZsiM6dYbPNFAiyNgXCelx9Ney4Y8w6Wpy3rXyk3FRvjbn11rGjXz6YRV8jBYLUpEBYj5YtYyrq55/Dz38eUwFFsu3RR6OfVkO3xmyoVCoWp61cmb/XlMKmQNiAHXaIFcyPPQajRiVdjZSa1atj7KBz52jFnk8VFTHLaObM/L6uFC4FQgZOPRV+8hO46CJdYkt23Xdf/ExdcUXDt8ZsKK1YltoUCBkwi72Y27WLqaiff550RVIKVq6M20Q9e8bPVb5tvz20aqVAkDUUCBnackuYMCEur889N+lqpBSMHx8rk7OxNWZDNGsGvXurhYWsoUCoh/33j3UJN98MkyYlXY0Usy+/hMsvh113hUMP3eDhOZNKxYI49e4SUCDU25AhsNNOMQC4cGHS1Uixuumm6Gqara0xGyqVioaOb72VXA1SOBQI9dSiBfzpT7G72rHHxvJ/kfr47LMIgh/8APbbL9laNLAsNSkQGqB7d7juOnjySRgxIulqpNhce23siHbllUlXEh1+mzbVOIIEBUIDnXACHHkkXHIJvPBC0tVIsfjgg3gTcfjhsMsuSVcT+4DssIOuECQoEBrIDP7wh9hs5OijYenSpCuSYjB0aPys5GJrzIbS3ghSTYHQCFtsAXfcAXPnwi9/mXQ1UugWLoxmicceCz16JF3NGqlU9Op6992kK5GkKRAaaa+9ojHZrbfCPfckXY0UsiFDYhLCFVckXcna1ApbqikQsuDSS2G33aLFxdtvJ12NFKI5c+CWW2DgwNxujdkQffrE3woEUSBkQfPmcOed0ajsmGM0FVW+7tJL87M1ZkNsthl06aJAEAVC1nTpAjfeGBudXHVV0tVIIZk2Ldqon3VW7HlQiCoqNPVUFAhZdcwx8LOfxT3i555LuhopFJdcErug5WNrzIZKpWJyxCefJF2JJEmBkGWjR8O220YwfPpp0tVI0v7xj9iC9YILYlZaoapesTx1arJ1SLIUCFm22WbR2uKdd+D005OuRpJUvTVm+/aFPy1ZLSwEFAg5sfvucNllsUbhjjuSrkaS8thj8PTTsedBmzZJV7N+W28diyw1jlDezIuo721lZaVXVVUlXUZGVq2CffaJAcWpU2PQWcrH6tWw887w0Uexh0a+d0NriEMOiSvb6dOTrkSyzcxedPfKDR2nK4Qcado0rg6aNInxBG1kXl7uvz/ebSexNWZDpVIwYwYsW5Z0JZIUBUIObbddbKbz/POxSlXKQ/XWmD16xJuBYpFKxZWtrhDKlwIhx446Cn7+82h1/MwzSVcj+VC91epvf5vM1pgNpRYWokDIg+uuizGEn/0M/vOfpKuRXKreGnPnneEnP0m6mvrp1CnWSygQypcCIQ822SSmoi5eDL/4hfavLWV/+APMn5/81pgNYRZ9jRQI5UuBkCc77wy/+U10RL3ttqSrkVz47LO4NbjffrE9ZjFKpeCVVzQJolwpEPJo0CDYd18480yYPTvpaiTbRo2CJUuKu5dVRUXc9nrjjaQrkSRkFAhmdpCZzTSzOWZ2UR1f387MHjezV8zsKTPrWONrw8zsNTObYWbXmcWFtJntZGbT0+f87+OlrGnTGHBs2TJ2WVu+POmKJFs+/DC2xjzsMNh116SraTitWC5vGwwEM2sKjAYOBnoAA8ys9n5PI4AJ7t4bGAJcnX7uHsCeQG+gF7AzsHf6OTcBpwLd0n8Oauw3Uww6doSxY6GqKloiS2kYOjRuGRXS1pgN8Z3vQKtWCoRylckVwi7AHHef6+7LgYnAYbWO6QE8nv74yRpfd6AV0AJoCTQH3jOzbYBN3f05j6XSE4Aim5PRcIcfHpvpDBsGTzyRdDXSWAsXwvXXR7fbXr2SrqZxmjWDHXdUC4tylUkgdADm1/h8QfqxmqYBR6Q/PhzYxMy2dPfniIBYnP4zxd1npJ+/YAPnLGkjR8a7sb594eyztZ9tsVq2DI4/PhZ0XX550tVkRyoV7VY0G678ZBIIdd3br/2jcj6wt5m9TNwSWgisNLOuwA5AR+If/P3MbK8MzxkvbnaqmVWZWdWSJUsyKLc4tGkD//d/8a7yhhtincJ558H77yddmWTqyy9jrcETT8T2mKXSryqVin0R5s1LuhLJt0wCYQGwbY3POwKLah7g7ovcvZ+7p4DB6cc+Ia4Wnnf3pe6+FJgM7JY+Z8f1nbPGuW9290p3r2zXrl2G31Zx6NAhxhNmzowVzaNGQefOcOGF8MEHSVcn6/PVV3Hr729/g3Hj4Nhjk64oe6oHlnXbqPxkEggvAN3MrLOZtQD6Aw/VPMDM2ppZ9bkuBsalP36HuHJoZmbNiauHGe6+GPjMzHZLzy46DngwC99PUfr2t2NtwowZ0K8fDB8ewTB4cHTLlMLy1VdwxBHw6KPwxz9Ga5JS8t3vxow4DSyXnw0GgruvBM4ApgAzgHvc/TUzG2Jmh6YP2weYaWazgPbAlenH7wPeBKYT4wzT3P3h9NdOA8YCc9LHTM7Kd1TEuneH22+H116DH/0Irr462glceqlaXhSK5cvjau6vf41VySedlHRF2deqVTTmUyCUH+2HUMBeey3aJ997b+zEdu65sVH7ZpslXVl5WrEC/ud/YNIkuPFGOO20pCvKneOPjw1+Fi9OuhLJBu2HUAJ69oxWF9OmRTuEyy6LW0lXXhlz3iV/VqyAAQMiDK6/vrTDAGIc4d13FQjlRoFQBHr3hj//GV58Eb73PbjkkgiGa66BpUuTrq70rVwZs8Huvx+uvRbOOCPpinJPrbDLkwKhiFRUwEMPwb//He0RLr44gmHECPjii6SrK02rVsFxx8WV2ogRsWakHPTpE38rEMqLAqEI7bxzDGo+9xzstFM0zevSJd69avvD7Fm1KmYQ3XVXtKY477ykK8qfTTeN2W8KhPKiQChiu+0WUx+ffTZaJpx7bvwSX399LJqShlu1KmYQ3XFHjNlccEHSFeVfRYXWIpQbBUIJ2HPPWPX897/H1NVf/hK6do2ZMF99lXR1xWf16ug1NX587IX9q18lXVEyUqlYrfzxx0lXIvmiQCghe+0FTz0VrRQ6d4bTT4du3eDmm9VqO1OrV8euduPGxfqPX/866YqSU71ieerUZOuQ/FEglKB994Wnn4555B06wMCB0Uhv3LiYPil1c48Q/eMfY5V4qTSrayjtjVB+FAglygwOOAD++U+YPBnatYt74ttvH7dCtEXi2txjJ7sxY6KX1G9+U3x7Imdb+/bwzW9qHKGcKBBKnBkcdBD861/w8MOw+eYxc6ZHD7jzzhg8LXfucM45MHo0nH9+tAwp9zColkrpCqGcKBDKhFn0R6qqggcegI02WrOhy8SJce+8HLlHCPz+97HGYNgwhUFNqVQ0XdQ6l/KgQCgzZrHv70svwX33RVfLAQNiNfR995VXMLjH7aGRI+N20ciRCoPaUqn4mZg+PelKJB8UCGWqSZNo4fzKK3D33fFLf+SR8Q/ApEmlv1uWewwcDx8O//u/cYWgMPg6tbAoLwqEMtekSbRznj49xhS+/DL2ZNhppxhzKNVguOyyGCsYODAW8ikM6rbddrDFFgqEcqFAECBuHR19dLTcHj8ePv0UDj0UdtkFHnmktILhiitiFtHJJ8fivSb6LVgns+hrpEAoD/pVkLU0axbN3N54I9YtfPABHHII7L57rGso9mD47W9jfcHPfx4b3CgMNqyiIm4tag1L6dOvg9SpWTM44QSYNSsWai1eDAceCN//Pjz+eHEGwzXXxMrjY4+NvawVBplJpaIFyhtvJF2J5Jp+JWS9mjePWyuzZ8NNN8Hbb8MPfgD77BO9k4rF8OHRLvzoo+HWW+MWmWRGK5bLhwJBMtKiRfT4mT07BmFnz45Q2H//6LZayK69NrqV9u8f4yMKg/r5znegdWsFQjlQIEi9tGoVO4a9+SaMGhWD0N//Pvzwh7E/Q6G57rpoC37kkXD77XErTOqnaVPYcUe1sCgHCgRpkNat4ayzYO7c2Els6lTYYw/o2xdeeCHp6sLo0VFjv34xpVZh0HCpVPw/LqeFi+VIgSCNstFGsZPYvHmxq9i//x1TVX/842TfUY4ZE1cyhx0WO541b55cLaUglYqpyPPmJV2J5JICQbKiTZu4Tz9vXuww9o9/xOK2ww+HadPyW8sf/winnRa9m+65J8Y/pHGqB5Z126i0KRAkqzbZJHYYmzcvdht78slY2HTkkfDqq7l//XHjYrezvn2jN5PCIDt69YpbbhpYLm0KBMmJzTaLOf9vvRU7j02ZEg30+veP7pm5MH58TJE98EC4/35o2TI3r1OOWrWKlukKhNKmQJCc2nzzaBXx1ltx5fDXv0LPntF6e9as7L3OHXfEQrr994/mfK1aZe/cElKpuGVUjIsSJTMKBMmLb3wj2kbMmxdjDZMmwQ47wPHHw5w5jTv3XXfFefbdFx58MGZASfalUvD++7FqXUqTAkHyqm3baCExb17sUnbvvbGt50knNWwGyz33xNXG978PDz0Us54kN9QKu/QpECQRW20V6xfmzo3Nae68E7p3jwHht9/O7Bz33x+tKPbcE/7yl5jpJLmz447xtwKhdCkQJFFbbx2tJebOjdYY48dDt26xac2CBet+3gMPxAD1rrvGuMTGG+ev5nK16abQtasCoZQpEKQgfPOb0SNpzpyYKTR2LHz723H1sGjR2sc+/HBs6lNZCZMnx1RXyY+KCq1FKGUKBCko224bm9bMnh0DxWPGRDCccw68+25cDRxxRKxtePTReNcq+ZNKxYyx//wn6UokFxQIUpC22w5uvjmmph59dFw9dOkSfYl6947NejbbLOkqy0/1iuWpU5OtQ3JDgSAFrXNnuOWW2JzlqKOi5fZjj8X6Bsk/tbAober/KEWha1e47bakq5CttoIOHTSwXKp0hSAi9ZJKKRBKlQJBROollYpbeF98kXQlkm26ZSQi9ZJKxUY5r7wCu+2WdDXZU1UV05xXrky6krrdey906pTb11AgiEi91GxhUUqBMHhwXPnssUfSldQtH3uBZxQIZnYQ8HugKTDW3a+p9fXtgHFAO+Aj4Bh3X2Bm+wLX1jh0e6C/uz9gZvsBI4AWwIvASe5eoNksItW+9S3YYovSGkd4+eWYvXbVVXDxxUlXk5wNjiGYWVNgNHAw0AMYYGY9ah02Apjg7r2BIcDVAO7+pLv3cfc+wH7AF8BjZtYEGE+EQy/gbeD4LH1PIpJDZqU3sDxsWKx4P+20pCtJViaDyrsAc9x9rrsvByYCh9U6pgfwePrjJ+v4OsBPgcnu/gWwJfCVu1d3xP8bcER9ixeRZFRUxBjCihVJV9J4c+dG19yBA7W+JZNA6ADMr/H5gvRjNU1jzT/ohwObmNmWtY7pD9yV/vgDoLmZVaY//ymwbaZFi0iyUilYvjx3u9/l08iRcX/+7LOTriR5mQSC1fFY7T2Tzgf2NrOXgb2BhcB/xwPMbBvgu8AUAHd3IiCuNbN/A5/VPH6tFzc71cyqzKxqyZIlGZQrIrlWvWK52G8bLVkS+3Afe2wsuCt3mQTCAtZ+994RWKv/pLsvcvd+7p4CBqcf+6TGIUcBk9x9RY3nPOfu33f3XYCngdl1vbi73+zule5e2a5du4y+KRHJre7dYzOiYm9hcf31sGwZDBqUdCWFIZNAeAHoZmadzawF8c7+oZoHmFnb9EAxwMXEjKOaBrDmdlH1c7ZK/90SuBAYU//yRSQJTZvGhjnFfIWwdCnccAMcdljs2icZBEJ6KugZxO2eGcA97v6amQ0xs0PTh+0DzDSzWUB74Mrq55tZJ+IK4++1Tj3IzGYArwAPu/sTjftWRCSfUqnoerp6ddKVNMzYsdHG+8ILk66kcFjczi8OlZWVXlVVlXQZIkL8g3rKKbF3RdeuSVdTPytWxD4bnTrB008nXU3umdmL7l65oePUy0hEGqSYW2FPnAjz5+vqoDYFgog0SK9e0KxZ8Y0juMdCtF69oG/fpKspLOplJCIN0rIl9OxZfIHwyCPw6qswYUKsupY1dIUgIg2WSsUtoyIaimTo0Ni7u3//pCspPAoEEWmwiopY3LVo0YaPLQTPPQfPPAPnngvNmyddTeFRIIhIgxXbiuWhQ6NT68knJ11JYVIgiEiD7bhj3IcvhkB44w148EE44wzYeOOkqylMCgQRabBNNok1CMUQCMOHQ+vWsSua1E2BICKNUlFR+GsRFi6E22+HE08EtURbNwWCiDRKKgVvvw0ffZR0Jes2ahSsWhWDybJuCgQRaZTqgeWpU5OtY10+/hj+8Ac46ijo0iXpagqbAkFEGqXQW1jcdBN89hlccEHSlRQ+BYKINEq7dtCxY2EOLH/5Jfz+9/DDH64JLlk3BYKINFoqVZiBMGECvPeemthlSoEgIo2WSsHMmfD550lXssaqVTHVtLIS9t036WqKgwJBRBotlYqNcl55JelK1pg0CebMibEDNbHLjAJBRBqtoiL+LpTbRu7RpqJrV+jXL+lqiofaX4tIo227LXzjG4UTCE8+CVVVMGZM7P8smdEVgog0mllhDSwPGwbt28PxxyddSXFRIIhIVlRUwPTpsV9xkqZOhSlT4KyzoFWrZGspNgoEEcmKVAqWL4fXX0+2jmHDopvpaaclW0cxUiCISFYUwt4I8+bB3XfDwIGw+ebJ1VGsFAgikhXdusFGGyXbwuJ3v4tB5HPOSa6GYqZAEJGsaNoU+vRJ7gphyRIYNw6OOQY6dEimhmKnQBCRrEmlYlB39er8v/YNN8CyZTBoUP5fu1QoEEQka1IpWLoU3nwzv6/7+ecRCIcdBjvskN/XLiUKBBHJmqRaYY8dGxv0qIld4ygQRCRrevaE5s3zO46wYgWMHAnf+x7svnv+XrcUqXWFiGRNy5YRCvkMhIkT4Z13YPTo/L1mqdIVgohkVXULC/fcv5Z7LETr2RP69s3965U6BYKIZFVFRUwBXbgw9681eTK8+mq0uG6if80aTf8JRSSr8rlieejQ6LQ6YEDuX6scKBBEJKt23DG6n+Y6EJ5/Hp5+Gs49NwaypfEUCCKSVRtvHG0scj31dOhQ2GILOPnk3L5OOVEgiEjWVVTk9grhjTfgwQfh9NMjgCQ7FAgiknWpVEwF/fDD3Jx/+PCY4nrmmbk5f7lSIIhI1lUPLE+dmv1zL1oEt98OJ54IW22V/fOXMwWCiGRdLltYjBoFq1bBeedl/9zlToEgIlnXtm1MB832OMLHH8OYMXDUUdClS3bPLRkGgpkdZGYzzWyOmV1Ux9e3M7PHzewVM3vKzDqmH9/XzKbW+POlmf0k/bX9zeyl9OPPmlnX7H5rIpKk6hXL2TRmDHz2WSxEk+zbYCCYWVNgNHAw0AMYYGY9ah02Apjg7r2BIcDVAO7+pLv3cfc+wH7AF8Bj6efcBPws/bU/AZdk4fsRkQKRSsHMmdGaOhu+/DJuFx1wwJpbUpJdmVwh7ALMcfe57r4cmAgcVuuYHsDj6Y+frOPrAD8FJrv7F+nPHdg0/fFmwKL6FC4iha2iInoNTZuWnfNNmADvvacW17mUSSB0AObX+HxB+rGapgFHpD8+HNjEzLasdUx/4K4an58MPGJmC4BjgWsyLVpECl82W1isWgUjRsBOO8F++zX+fFK3TALB6nisdh/D84G9zexlYG9gIbDyvycw2wb4LjClxnPOAfq6e0fgVmBknS9udqqZVZlZ1ZIlSzIoV0QKQceOsOWW2QmEBx6A2bPj6sDq+hdJsiKTQFgAbFvj847Uur3j7ovcvZ+7p4DB6cc+qXHIUcAkd18BYGbtgB3d/V/pr98N7FHXi7v7ze5e6e6V7dq1y+R7EpECYJadgWX3aFPRtSv065ed2qRumQTCC0A3M+tsZi2IWz8P1TzAzNqaWfW5LgbG1TrHANa+XfQfYDMz657+/ABgRn2LF5HCVlEB06fD8uUNP8dTT8ELL8D550PTplkrTeqwwUBw95XAGcTtnhnAPe7+mpkNMbND04ftA8w0s1lAe+DK6uebWSfiCuPvtc55CnC/mU0jxhAGZeH7EZECkkrFFpevv97wcwwdGiuSjzsue3VJ3TLaQtPdHwEeqfXYpTU+vg+4bx3PfYuvD0Lj7pOASfWoVUSKTM2B5T596v/8qVNhyhS48kpo3Tq7tcnXaaWyiORMt27Qpk3DW1gMHx7dTE87Lbt1Sd0UCCKSM02axJVBQwaW33oL7r4bBg6MfQ8k9xQIIpJTqVQsTlu9un7P+93vIlDOOSc3dcnXKRBEJKdSKVi6FObMyfw5S5bALbfAMcdAh6+NQEquKBBEJKca0gr7hhtg2TIYpLmHeaVAEJGc6tkTmjfPfBzh88+Y1qEMAAAGLElEQVQjEA49FHbYIbe1ydoUCCKSUy1aQK9emQfCLbfARx+piV0SFAgiknPVLSy8dhe0WlasiMHk730P9qizmY3kkgJBRHKuogI++AAWLFj/cXffDe+8o6uDpCgQRCTnMmmF7Q7DhsWYQ9+++alL1qZAEJGc6907up+uLxAmT45GeIMGxfoDyT/9ZxeRnNt4Y+jeff1TT4cOjT0UBgzIX12yNgWCiORFRcW6rxCefx6efhrOPTdmJUkyFAgikhepFMyfDx9++PWvDRsW/YpOOSX/dckaCgQRyYt1DSzPnBlbZJ5+etxakuQoEEQkL9bVwmL4cGjZEs48M/81ydoUCCKSF1tuCd/61tpXCIsWwe23wwknxK5okiwFgojkTfWK5WqjRsHKlbFfsiRPgSAieZNKwaxZ0Q77k09gzBg48kjo0iXpygQy3FNZRCQbKipiRfK0afDss/DZZ2pTUUh0hSAieVM9sPz883G76IAD1jwmydMVgojkTYcO0LYtXH11rEe4446kK5KadIUgInljFlcEH34Yt4/22y/piqQmBYKI5FVFRfx94YUREFI4dMtIRPLqxBOjm+kRRyRdidSmQBCRvOreHa66KukqpC66ZSQiIoACQURE0hQIIiICKBBERCRNgSAiIoACQURE0hQIIiICKBBERCTN3D3pGjJmZkuAtxv49LbAB1ksJ1tUV/2orvpRXfVTqnVt5+7tNnRQUQVCY5hZlbtXJl1HbaqrflRX/aiu+in3unTLSEREAAWCiIiklVMg3Jx0AeuguupHddWP6qqfsq6rbMYQRERk/crpCkFERNajLALBzA4ys5lmNsfMLkq6HgAzG2dm75vZq0nXUpOZbWtmT5rZDDN7zczOSromADNrZWb/NrNp6bquSLqmmsysqZm9bGZ/SbqWamb2lplNN7OpZlaVdD3VzGxzM7vPzN5I/5ztXgA1fSf936n6z6dmdnbSdQGY2Tnpn/lXzewuM2uVs9cq9VtGZtYUmAUcACwAXgAGuPvrCde1F7AUmODuvZKspSYz2wbYxt1fMrNNgBeBnxTAfy8D2rj7UjNrDjwLnOXuzydZVzUzOxeoBDZ19x8lXQ9EIACV7l5Q8+rNbDzwjLuPNbMWwEbu/nHSdVVL/5uxENjV3Ru67ilbtXQgftZ7uPsyM7sHeMTdb8vF65XDFcIuwBx3n+vuy4GJwGEJ14S7Pw18lHQdtbn7Ynd/Kf3xZ8AMoEOyVYGHpelPm6f/FMS7GTPrCBwCjE26lkJnZpsCewG3ALj78kIKg7T9gTeTDoMamgGtzawZsBGwKFcvVA6B0AGYX+PzBRTAP3DFwMw6ASngX8lWEtK3ZaYC7wN/c/eCqAsYBVwArE66kFoceMzMXjSzU5MuJq0LsAS4NX2LbayZtUm6qFr6A3clXQSAuy8ERgDvAIuBT9z9sVy9XjkEgtXxWEG8syxkZrYxcD9wtrt/mnQ9AO6+yt37AB2BXcws8VttZvYj4H13fzHpWuqwp7tXAAcDp6dvUyatGVAB3OTuKeBzoCDG9QDSt7AOBe5NuhYAM9uCuKPRGfgm0MbMjsnV65VDICwAtq3xeUdyeMlVCtL36O8H7nT3PyddT23pWwxPAQclXArAnsCh6fv1E4H9zOyOZEsK7r4o/ff7wCTi9mnSFgALalzd3UcERKE4GHjJ3d9LupC0HwDz3H2Ju68A/gzskasXK4dAeAHoZmad0+nfH3go4ZoKVnrw9hZghruPTLqeambWzsw2T3/cmvhFeSPZqsDdL3b3ju7eifjZesLdc/YOLlNm1iY9KYD0LZkfAonPaHP3d4H5Zvad9EP7A4lOWKhlAAVyuyjtHWA3M9so/bu5PzGulxPNcnXiQuHuK83sDGAK0BQY5+6vJVwWZnYXsA/Q1swWAJe5+y3JVgXEO95jgenp+/UAv3L3RxKsCWAbYHx6BkgT4B53L5gpngWoPTAp/g2hGfAnd3802ZL+60zgzvQbtLnACQnXA4CZbUTMRhyYdC3V3P1fZnYf8BKwEniZHK5aLvlppyIikplyuGUkIiIZUCCIiAigQBARkTQFgoiIAAoEERFJUyCIiAigQBARkTQFgoiIAPD/JdtXlWa0PwYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"隐层2的节点数目对于准确率的影响\")\n",
    "pyplot.plot(acc_array.keys(), acc_array.values(), 'b-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "数目不是越多越好。。。160是逐渐缩小范围得来的。尝试下step对结果的影响，锁定节点数为【800,160】"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "0.9753\n",
      "784_800\n",
      "800_160\n",
      "0.9757\n",
      "784_800\n",
      "800_160\n",
      "0.98\n",
      "784_800\n",
      "800_160\n",
      "0.9803\n",
      "784_800\n",
      "800_160\n",
      "0.9815\n",
      "784_800\n",
      "800_160\n",
      "0.9778\n",
      "784_800\n",
      "800_160\n",
      "0.9782\n",
      "784_800\n",
      "800_160\n",
      "0.098\n",
      "评分最好的i和acc是：0.5,0.9815\n"
     ]
    }
   ],
   "source": [
    "\n",
    "acc_array = {}\n",
    "\n",
    "best_j = 0\n",
    "best_acc = 0\n",
    "steps = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]\n",
    "for j in range(len(steps)):\n",
    "    _step = steps[j]\n",
    "    _layer_nodes = [800,160] #[600,500,400,10]\n",
    "    accuraccy = train(step = _step, layer_nodes = _layer_nodes)\n",
    "    acc_array[j] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_j = j\n",
    "    \n",
    "print(\"评分最好的i和acc是：\"+ str(steps[best_j]) + \",\" + str(best_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step大小对于准确率的影响\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x25418da0>]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFMRJREFUeJzt3WuMXPd53/Hvs7uiRUukLIEbxRbpkNUNvJgypSV5DgQEbuUWlGtIfVEUEpAUDYKoL6I0RoIaSls4rvsuAdrGgJpasN1cmlhQnCZRErVKETvIBRLFJSVaIiWVLGtJS0YldYlslpB4e/rizEbr1V5md2f2zDnz/QCLmTlzds4Pi+WP/31mzkxkJpKkdhmpO4Akqfcsd0lqIctdklrIcpekFrLcJamFLHdJaiHLXZJayHKXpBay3CWphcbqOvCGDRty8+bNdR1ekhrp4MGDb2Tm+GL71VbumzdvZnJysq7DS1IjRcQr3ey36FgmIr4eEacj4oV57o+I+HJEHI+I70TE7UsNK0nqrW5m7r8G7Fvg/ruBmztfDwC/uvJYkqSVWLTcM/PPgbcW2OVe4Dey8jTwkYj4aK8CSpKWrhevlrkBeG3G7anONklSTXpR7jHHtjnfJD4iHoiIyYiYPHPmTA8OLUmaSy/KfQrYNOP2RuDUXDtm5iOZOZGZE+Pji76SR5K0TL0o98eBf9p51UwBvJOZf92Dx5UkLdOir3OPiG8AnwI2RMQU8IvAFQCZ+Z+BJ4DPAMeBc8BP9CusVi4TLlyA8+err9nXL1x4f7+Z3zPftsXuX8m2br4nEy5fnv+y221L3X85jxEBY2NwxRVzfy1031LvHx2d/3dAw2HRcs/M+xe5P4Gf7lmiRXzve/D22x/8R9/N5XK+pxePffHi3EXa6+vd7Hfx4tJ/5mqeiJX/5xGx+v++lvu9ExPwK7+y9J9Tm9V2hupyfeUr8PnP152i/0ZGYM2a6uuKKxa/vnbt3Nu7/f7p62Nj1T9qeP9y5vW5ti12/0q2LXb/yEh1fa7Lbrctdf+lPkbE+0V56dL7fyHN/Lp4ce7t3d7fi8e4eBHeffcH/4Kb6+ffj8vF9hkZmf/+kyfhy1+GL34Rrr0WdTSu3Pftgw0bquv9+CXqxy/q2NjSitc/q9tpeiwzNlb9Z6ze+Na34K67YP/+qh9UaVy5f+IT1ZckAezeXa3sn37acp/Jt/yV1Gjr1sGOHfDUU3UnGSyWu6TGK4pqLHP5ct1JBoflLqnxyhLeeQdeeqnuJIPDcpfUeEVRXT79dL05BonlLqnxbrmlehmkc/f3We6SGm9kBPbudeU+k+UuqRXKEo4cqWbvstwltURRVGcBHzhQd5LBYLlLaoW9e6uzgJ27Vyx3Sa1wzTWwdatz92mWu6TWKMuq3Ge+RfSwstwltUZRwFtvwbFjdSepn+UuqTXKsrp07m65S2qRrVth/Xrn7mC5S2qR6ZOZXLlb7pJapijg+efh+9+vO0m9LHdJrVKW1Vv/Tk7WnaRelrukVtm7t7oc9tGM5S6pVa67Dm691SdVLXdJrVMU1cp9mE9mstwltU5ZwhtvwIkTdSepj+UuqXU8mclyl9RC27fD1VcP99zdcpfUOqOjsGePK3dJap2igMOH4dy5upPUw3KX1EplCZcuDe/JTJa7pFaaPplpWOfulrukVhofh5tuGt65u+UuqbWKYng/mclyl9RaZQmvvw6vvFJ3ktVnuUtqraKoLodx7m65S2qtnTth7drhnLtb7pJaa2wMdu925T6viNgXES9HxPGIeGiO+z8eEd+OiGcj4jsR8ZneR5WkpStLePZZePfdupOsrkXLPSJGgYeBu4FtwP0RsW3Wbv8GeCwzdwH3Af+p10ElaTmKAi5cgEOH6k6yurpZue8Bjmfmicw8DzwK3DtrnwTWd65fA5zqXURJWr7pJ1WHbe4+1sU+NwCvzbg9Beydtc8XgT+JiJ8BrgI+3ZN0krRCP/zDsHnz8M3du1m5xxzbZp8ScD/wa5m5EfgM8JsR8YHHjogHImIyIibPnDmz9LSStAxlOXwr927KfQrYNOP2Rj44dvlJ4DGAzHwKuBLYMPuBMvORzJzIzInx8fHlJZakJSoKOHkSpqbqTrJ6uin3A8DNEbElItZQPWH6+Kx9XgXuAoiIrVTl7tJc0kAYxk9mWrTcM/Mi8CDwJPAi1atijkTElyLins5uPw/8VEQcBr4B/LPMYXw3B0mD6Lbb4Morh2vu3s0TqmTmE8ATs7Z9Ycb1o8CdvY0mSb2xZg3ccYcrd0lqnaKAgwfhvffqTrI6LHdJQ6Es4fx5eO65upOsDstd0lAYtidVLXdJQ+FjH4NNm4bnSVXLXdLQGKaTmSx3SUOjKODVV+HUELz7leUuaWhMz92HYTRjuUsaGrt2Va95t9wlqUU+9CG4/fbhmLtb7pKGSlHA5GT1AR5tZrlLGiplWX3k3uHDdSfpL8td0lCZ/mSmts/dLXdJQ2XTpuqEprbP3S13SUMlolq9u3KXpJYpSzhxAk6frjtJ/1jukobOMMzdLXdJQ+eOO2BsrN1zd8td0tBZuxY++UlX7pLUOmUJzzwDFy/WnaQ/LHdJQ6ko4Nw5eOGFupP0h+UuaSi1/ZOZLHdJQ2nzZvihH2rv3N1ylzSUItr9yUyWu6ShVRRw7Bi8+WbdSXrPcpc0tNr8yUyWu6ShNTEBo6PtHM1Y7pKG1lVXwc6drtwlqXXKEvbvh0uX6k7SW5a7pKFWFHD2LBw9WneS3rLcJQ21tp7MZLlLGmo33ggbNrRv7m65Sxpq05/M5MpdklqmKOCll+Dtt+tO0juWu6ShNz1337+/3hy9ZLlLGnq7d8PISLvm7pa7pKG3bh3s2NGuuXtX5R4R+yLi5Yg4HhEPzbPPP4mIoxFxJCJ+u7cxJam/iqIay1y+XHeS3li03CNiFHgYuBvYBtwfEdtm7XMz8AvAnZm5HfhcH7JKUt+UJbzzTvXEaht0s3LfAxzPzBOZeR54FLh31j4/BTycmW8DZObp3saUpP4qiuqyLXP3bsr9BuC1GbenOttmugW4JSL+KiKejoh9vQooSavhllvg2mvbM3cf62KfmGNbzvE4NwOfAjYCfxEROzLzb37ggSIeAB4A+PjHP77ksJLULyMjsHfvcK3cp4BNM25vBE7Nsc8fZOaFzPw/wMtUZf8DMvORzJzIzInx8fHlZpakvihLOHKkmr03XTflfgC4OSK2RMQa4D7g8Vn7/D7wdwEiYgPVmOZEL4NKUr8VBWTCgQN1J1m5Rcs9My8CDwJPAi8Cj2XmkYj4UkTc09ntSeDNiDgKfBv4l5nZwk8llNRme/dW7zXThrl7ZM4en6+OiYmJnJycrOXYkjSf7dth82b44z+uO8ncIuJgZk4stp9nqErSDGVZPala07q3Zyx3SZqhKOCtt+DYsbqTrIzlLkkztOWTmSx3SZph61ZYv775r3e33CVphumTmVy5S1LLlCU8/zycPVt3kuWz3CVplqKo3vq3ySczWe6SNMvevdVlk0czlrskzXLddXDrrc1+UtVyl6Q5lGW1cm/qyUyWuyTNoSjgjTfgREPfAtFyl6Q5NP1kJstdkuawfTtcfXVz5+6WuyTNYXQU9uxx5S5JrVMUcPgwnDtXd5Kls9wlaR5lCZcuQRM/esJyl6R5TJ/M1MS5u+UuSfMYH4ebbmrm3N1yl6QFFEUzP5nJcpekBZQlvP46vPJK3UmWxnKXpAUURXXZtLm75S5JC9i5E9aubd7c3XKXpAWMjcHu3a7cJal1yhKefRbefbfuJN2z3CVpEUUBFy7AoUN1J+me5S5Ji2jiO0Ra7pK0iOuvhy1bmjV3t9wlqQtF4cpdklqnLOHkSZiaqjtJdyx3SerC9MlMTVm9W+6S1IXbboMrr2zO3N1yl6QurFkDd9zhyl2SWqcsq9e6v/de3UkWZ7lLUpeKoir2556rO8niLHdJ6lKTTmay3CWpSx/7GGza1IwnVbsq94jYFxEvR8TxiHhogf3+cURkREz0LqIkDY6ybMnKPSJGgYeBu4FtwP0RsW2O/dYB/wLY3+uQkjQoigJefRVOnao7ycK6WbnvAY5n5onMPA88Ctw7x37/DvgloEFviilJSzM9dx/00Uw35X4D8NqM21OdbX8rInYBmzLzjxZ6oIh4ICImI2LyzJkzSw4rSXXbtat6zXsbyj3m2Pa3nwMeESPAfwB+frEHysxHMnMiMyfGx8e7TylJA+JDH4Lbbx/8uXs35T4FbJpxeyMwc9q0DtgB/FlEfBcogMd9UlVSWxUFTE5WH+AxqLop9wPAzRGxJSLWAPcBj0/fmZnvZOaGzNycmZuBp4F7MnOyL4klqWZlWX3k3uHDdSeZ36LlnpkXgQeBJ4EXgccy80hEfCki7ul3QEkaNNPvEDnIc/exbnbKzCeAJ2Zt+8I8+35q5bEkaXBt2lSd0PTUU/Dgg3WnmZtnqErSEkVUq/dBXrlb7pK0DGUJJ07A6dN1J5mb5S5JyzDoc3fLXZKW4Y47YGxscF/vbrlL0jKsXVudrerKXZJapijgmWfg4sW6k3yQ5S5Jy1SWcO4cvPBC3Uk+yHKXpGWaflJ1EOfulrskLdPmzXD99YM5d7fcJWmZpk9mcuUuSS1TlnDsGLz5Zt1JfpDlLkkrMKgnM1nukrQCExMwOmq5S1KrXHUV7Nw5eHN3y12SVqgsYf9+uHSp7iTvs9wlaYWKAs6ehaNH607yPstdklaoLKvLQRrNWO6StEI33ggbNgzWk6qWuySt0CCezGS5S1IPFAW89BK8/XbdSSqWuyT1wPTcff/+enNMs9wlqQd274aRkcGZu1vuktQD69bBjh2DM3e33CWpR4qiGstcvlx3EstdknqmLOGdd6onVutmuUtSjwzSO0Ra7pLUI7fcAtdeOxhzd8tdknpkZKRavbtyl6SWKQo4cqSavdfJcpekHipLyIQDB+rNYblLUg/t2VO910zdc3fLXZJ66JprYNu2+ufulrsk9dj0k6qZ9WWw3CWpx8oS3noLjh2rL4PlLkk9Nn0yU51z967KPSL2RcTLEXE8Ih6a4/6fi4ijEfGdiPjTiPiR3keVpGbYuhXWr6937r5ouUfEKPAwcDewDbg/IrbN2u1ZYCIzdwLfBH6p10ElqSlGRmDv3sFfue8Bjmfmicw8DzwK3Dtzh8z8dmae69x8GtjY25iS1CxlCc8/D2fP1nP8bsr9BuC1GbenOtvm85PAf19JKElquqKo3vq3rpOZuin3mGPbnC/wiYgfAyaAX57n/gciYjIiJs+cOdN9SklqmL17q8u65u7dlPsUsGnG7Y3Aqdk7RcSngX8N3JOZ7831QJn5SGZOZObE+Pj4cvJKUiNcdx3cemt9c/duyv0AcHNEbImINcB9wOMzd4iIXcBXqIr9dO9jSlLzlGVV7nWczLRouWfmReBB4EngReCxzDwSEV+KiHs6u/0ycDXwOxHxXEQ8Ps/DSdLQKAp44w04cWL1jz3WzU6Z+QTwxKxtX5hx/dM9ziVJjVeW1eVTT8GNN67usT1DVZL6ZPt2uPrqep5UtdwlqU9GR6u3AK7jSVXLXZL6qCjg8GE4d27xfXvJcpekPipLuHQJJidX97iWuyT1UV0nM1nuktRH4+Nw002rP3e33CWpz8py9T+ZyXKXpD4rCnj9dXjlldU7puUuSX02fTLTas7dLXdJ6rNPfAI+/OHVnbtb7pLUZ2NjsHu3K3dJap2igGefhXffXZ3jWe6StArKEi5cgEOHVud4lrskrYKiqC5Xa+5uuUvSKrj+etiyZfXm7pa7JK2SonDlLkmtU5Zw8iRMTfX/WJa7JK2S1Zy7W+6StEpuuw0++1lYv77/x+rqM1QlSSu3Zg384R+uzrFcuUtSC1nuktRClrsktZDlLkktZLlLUgtZ7pLUQpa7JLWQ5S5JLRS5mh/HPfPAEWeA5X5c7AbgjR7G6bcm5W1SVmhW3iZlhWblbVJWWFneH8nM8cV2qq3cVyIiJjNzou4c3WpS3iZlhWblbVJWaFbeJmWF1cnrWEaSWshyl6QWamq5P1J3gCVqUt4mZYVm5W1SVmhW3iZlhVXI28iZuyRpYU1duUuSFtC4co+IfRHxckQcj4iH6s6zkIj4ekScjogX6s6ymIjYFBHfjogXI+JIRPxs3ZnmExFXRsQzEXG4k/Xf1p2pGxExGhHPRsQf1Z1lIRHx3Yh4PiKei4jJuvMsJiI+EhHfjIiXOr+/Zd2Z5hIRt3Z+ptNf34uIz/XteE0ay0TEKPC/gL8PTAEHgPsz82itweYRET8KnAV+IzN31J1nIRHxUeCjmXkoItYBB4F/NIg/24gI4KrMPBsRVwB/CfxsZq7S58ovT0T8HDABrM/Mz9adZz4R8V1gIjMb8brxiPh14C8y86sRsQb4cGb+Td25FtLpspPA3sxc7vk+C2rayn0PcDwzT2TmeeBR4N6aM80rM/8ceKvuHN3IzL/OzEOd698HXgRuqDfV3LJytnPzis7XQK9SImIj8A+Br9adpU0iYj3wo8DXADLz/KAXe8ddwP/uV7FD88r9BuC1GbenGNACarKI2AzsAvbXm2R+nRHHc8Bp4H9m5sBm7fiPwOeBy3UH6UICfxIRByPigbrDLOLvAGeA/9IZeX01Iq6qO1QX7gO+0c8DNK3cY45tA71ia5qIuBr4XeBzmfm9uvPMJzMvZeYngY3AnogY2LFXRHwWOJ2ZB+vO0qU7M/N24G7gpzvjxUE1BtwO/Gpm7gL+HzDoz8WtAe4Bfqefx2lauU8Bm2bc3gicqilL63Tm178L/FZm/re683Sj8yf4nwH7ao6ykDuBezqz7EeBvxcR/7XeSPPLzFOdy9PA71GNQwfVFDA14y+3b1KV/SC7GziUmf+3nwdpWrkfAG6OiC2d//3uAx6vOVMrdJ6k/BrwYmb++7rzLCQixiPiI53ra4FPAy/Vm2p+mfkLmbkxMzdT/c5+KzN/rOZYc4qIqzpPqNMZb/wDYGBf7ZWZrwOvRcStnU13AQP3IoBZ7qfPIxmo/qRpjMy8GBEPAk8Co8DXM/NIzbHmFRHfAD4FbIiIKeAXM/Nr9aaa153AjwPPd2bZAP8qM5+oMdN8Pgr8eucVByPAY5k50C8vbJDrgd+r/q9nDPjtzPwf9UZa1M8Av9VZ8J0AfqLmPPOKiA9Tvdrvn/f9WE16KaQkqTtNG8tIkrpguUtSC1nuktRClrsktZDlLkktZLlLUgtZ7pLUQpa7JLXQ/wcwhMu6/UJAxAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"step大小对于准确率的影响\")\n",
    "pyplot.plot(acc_array.keys(), acc_array.values(), 'b-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "发现运行同样数据，每次的结果也不尽相同，可能和随机的初始矩阵值也有关系。记录当前的最佳step = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "尝试添加第三个隐层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "160_50\n",
      "0.9797\n",
      "784_800\n",
      "800_160\n",
      "160_110\n",
      "0.9766\n",
      "784_800\n",
      "800_160\n",
      "160_150\n",
      "0.098\n",
      "784_800\n",
      "800_160\n",
      "160_150\n",
      "0.9767\n",
      "784_800\n",
      "800_160\n",
      "160_200\n",
      "0.9788\n",
      "784_800\n",
      "800_160\n",
      "160_300\n",
      "0.098\n",
      "784_800\n",
      "800_160\n",
      "160_400\n",
      "0.098\n",
      "评分最好的i和acc是：50,0.9797\n"
     ]
    }
   ],
   "source": [
    "acc_array = {}\n",
    "\n",
    "best_j = 0\n",
    "best_acc = 0\n",
    "nodes = [50,110,150,150,200,300,400]\n",
    "for j in range(len(nodes)):\n",
    "    node_count = nodes[j]\n",
    "    _layer_nodes = [800,160,node_count] #[600,500,400,10]\n",
    "    accuraccy = train(step = 0.5, layer_nodes = _layer_nodes)\n",
    "    acc_array[j] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_j = j\n",
    "    \n",
    "print(\"评分最好的i和acc是：\"+ str(nodes[best_j]) + \",\" + str(best_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "隐层2的节点数目对于准确率的影响\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x27d21b00>]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3WuQVOed3/HvnxkGAQIhixHCgLjIyBZIQqDhjoB+1klJzq6UF9nYqnVS2dpavVnb6/JuUt7EpU20tW+8tUkqVc7GKtvxbrK2SvFmE5VLidbrM9xBYhC6IUSM0A2hy0jCSCAQF/3z4pm2uAwzPT3d85zL71NFDTNz6P631M/vPP0/zznH3B0RESmXcakLEBGR1lO4i4iUkMJdRKSEFO4iIiWkcBcRKSGFu4hICSncRURKSOEuIlJCCncRkRLqTPXE06dP93nz5qV6ehGRQtqzZ8877t493HbJwn3evHn09fWlenoRkUIys1ca2W7YtoyZ/cDM3jaz5y7zezOz/2RmB83sGTNbNtJiRUSktRrpuf8QuHOI398FLBz4cx/wF6MvS0RERmPYcHf3LcB7Q2xyD/BXHu0CppnZzFYVKCIiI9eK1TKzgNfO+/7wwM9ERCSRVoS7DfKzQS8Sb2b3mVmfmfX19/e34KlFRGQwrQj3w8Cc876fDRwZbEN3f9Dde9y9p7t72JU8IiLSpFaE+yPAPx9YNbMKOObub7TgcUVEpEnDrnM3sx8DG4HpZnYY+GNgPIC7/xfgUeALwEHgQ+C321UswLZt8LOfQUcHdHa25murHsMGa1BJZbnDxx/HP+fOXfh1uJ+16t/Mmwe33JL6v4SkMGy4u/u9w/zegd9rWUXD2LkTHnhgrJ5tZMaNG/lO4YtfhG99K3XlUvfSS/CbvwnHj48+dPNwe+LJk+Hdd2HChNSVyFizVDfI7unp8WbPUK3PiM6dg7Nn8/11qN89/TT098c/HR0t/g8sTfnzP4c//MMY8OPHxx12R0f8erm/N/P7djzmxb/ftQu+9jXYtAk2bEj9X1Zaxcz2uHvPcNslu/zAaJh90grp6kpdTfN+9CP4rd+KIb9M5/XmQpbB5z4HDz+cupLRW7gQvv71+JoU7tWjq0ImVKvFr1mWtg6JzpyBLVsghNSVtMa0aXD77Xp/VZXCPaGZM+MsUYMvH/r6Yq+9LOEOcQLx+ONw4kTqSmSsKdwTCyHOFs+cSV2J1HeyGzcmLaOlQojvre3bU1ciY03hnlgIcVa1e3fqSiTL4Lbb4JprUlfSOuvWxZVZ+nRYPQr3xOqzRA2+tE6dirPb+nGQspg8GVat0vurihTuiV1zTZwt9vamrqTadu6Ejz4qV7+9LgTYsweOHUtdiYwlhXsOhBBnjadOpa6kurIsLq1dvz51Ja1Xq8XzQrZsSV2JjCWFew6EEGeNO3emrqS6sgx6emDq1NSVtN6qVXDFFWrNVI3CPQfuuCPOGjX40jh+HJ54opwtGYjBvnat3l9Vo3DPgalTYflyDb5Utm2Ll4Qoa7hDfG3PPBMvdSHVoHDPiRDi7PGDD1JXUj1ZFi9jsWZN6krap74KaPPmtHXI2FG450QIcfa4bVvqSqony2D1apg0KXUl7dPTA1deqU+HVaJwz4k1a+LsUYNvbB09Ck8+We6WDMQrXK5fr/dXlSjcc2LixDh71OAbW5s3x0tIlz3cIb7GAwfgyKA3wZSyUbjnSAiwdy+8917qSqojy2I7ZsWK1JW0X73vrhPmqkHhniMhxFmkDnqNnSyLS1GLfF+ARi1ZAldfrU+HVaFwz5EVK+IsUoNvbLz1FuzbV42WDMRzKTZu1PurKhTuOdLVFWeRGnxjo96eKNvFwoYSArz8crxXrJSbwj1nQoDnn4+zSmmvLIOrroKlS1NXMnZ096/qULjnTL1FoINe7Ve/t2hnIe8k3JxFi+Daa/X+qgKFe84sXRpnk5pZtdcrr8CLL1an315nFl9zlsWD91JeCvec0UGvsVGfuVYt3CG+5jfeiGvepbwU7jkUQpxVvvJK6krKq7cXurth8eLUlYw99d2rQeGeQ+q7t5d7DLZaDcZVcATccAPMmaP3V9lV8K2df4sXx1mlZlbtcfAgHD5czZYMfNJ37+2Nd2iSclK455AOerVXfadZ1XCH+NrffReefTZ1JdIuCvecCgFefx1+8YvUlZRPlsHs2fCZz6SuJB313ctP4Z5TGnzt8fHHsR0RQvyEVFVz5sSdm/ru5aVwz6nPfCbOLhXurbVvX7zVXJVbMnUhxIvUnT2buhJpB4V7TumgV3vUd5ZVup7M5YQA778fb1Yi5aNwz7EQ4J134LnnUldSHlkWlwJef33qStLbuDF+1afDclK455j67q119ixs2qSWTN2MGXHZrfru5aRwz7Hrr9dBr1bauze2IRTunwgBtm6F06dTVyKtpnDPuRDibFMHvUZP/fZLhQAnT8Ljj6euRFqtoXA3szvN7ICZHTSzbw7y++vNrNfM9prZM2b2hdaXWk31g15796aupPiyLLYhZsxIXUl+bNgQD96r9Vc+w4a7mXUA3wHuAhYB95rZoos2+xbwsLsvBb4E/OdWF1pVOujVGqdPw7Ztaslc7Oqr42Wm1forn0Zm7iuAg+5+yN1PAw8B91y0jQNTB/5+FXCkdSVW24wZcPPNCvfReuIJ+PBDhftgQoCdO+N/HymPRsJ9FvDaed8fHvjZ+f4t8GUzOww8Cny1JdUJoINerZBlsf2wYUPqSvInhPje2rEjdSXSSo2E+2AnaV98Oat7gR+6+2zgC8B/M7NLHtvM7jOzPjPr6+/vH3m1FaWDXqOXZbBsWWxDyIXWrYu3GtSnw3JpJNwPA3PO+342l7Zdfgd4GMDddwJXANMvfiB3f9Dde9y9p7u7u7mKK2jDhnjdcQ2+5nz4YWw7qCUzuClTYPlyvb/KppFw3w0sNLP5ZtZFPGD6yEXbvAr8GoCZ3UQMd03NW2TatHjQS4OvOTt2xLaDwv3yQoC+vrgyS8ph2HB397PAV4DHgP3EVTH7zOwBM7t7YLM/AH7XzJ4Gfgz8C3ddibyVdNCreVkW2w7r1qWuJL9CgHPn4rEdKYeG1rm7+6PufqO73+Dufzrws/vd/ZGBvz/v7mvdfYm73+buf9fOoqsoBDhzBrZvT11J8WQZrFwJV16ZupL8Wr0aJkzQp8My0RmqBaGDXs05dgx279ZZqcOZOBHWrNH7q0wU7gVx5ZVx9qnBNzJbt8ZLJqvfPrxaDZ5+Ot5+T4pP4V4g9YNex46lrqQ4siy2G1avTl1J/oUQ79m7eXPqSqQVFO4FEkKcheqgV+OyDNauhSuuSF1J/i1fDpMn69NhWSjcC2TVqhhSGnyNeeed2GZQS6YxXV1wxx16f5WFwr1ArrgizkI1+BpTby8o3BtXq8H+/fDmm6krkdFSuBdMCHE2+s47qSvJvyyLB6J7elJXUhz1HaGuEll8CveCqQ++TZuSllEIWQbr18P48akrKY6lS+Gqq/TpsAwU7gXT0xOvBaLBN7QjR+CFF9SSGamOjngPAb2/ik/hXjCdnXE2qsE3tHpbQeE+crUaHDoEr7ySuhIZDYV7AdVqcOAAvP566kryK8vi5X2XLEldSfGo714OCvcC0uAbXpbFneA4vcNHbPFi6O7Wp8Oi01u/gJYsibNSDb7BvfQSvPyyWjLNGjcu7hizLJ6xKsWkcC+g+uD7+c81+AZT3+npYmHNq9Vi2+/gwdSVSLMU7gUVArz6apylyoWyLN5Y/KabUldSXPVPPfp0WFwK94JS331w7jGQQog3xJbmLFwIs2Yp3ItM4V5Qn/scXHedBt/FXnghnjqvfvvomMX/hr29av0VlcK9oOqDTwe9LlTf2SncRy8E6O+HfftSVyLNULgXWAhxlvrCC6kryY/eXpg7F+bPT11J8dUPSOvTYTEp3AtMB70u9PHHMdzVb2+NuXNhwQK9v4pK4V5g8+fDvHkafHXPPAPvvaeWTCuFEC9Sd+5c6kpkpBTuBVc/6PXxx6krSU/r21svhHhbx717U1ciI6VwL7gQ4OjReI33qssy+Oxn4xI+aY2NG+NXLbktHoV7wemgV3TmTLzzkloyrTVzZjwZrOrvryJSuBfcpz8dZ6tVH3x79sDx4wr3dggh3pT99OnUlchIKNxLIATYsiXOXquqvnOrtxGkdUKAEydg9+7UlchIKNxLIIQ4a+3rS11JOlkGt94K06enrqR8NmyIS0vVdy8WhXsJ1GerVW3NnDoF27erJdMu11wTLzNd1fdXUSncS2D69GoPvl27YsAr3NsnBNixA06eTF2JNErhXhL1wXfqVOpKxl6WxWvcr1+fupLyCgE++gh27kxdiTRK4V4SIcRg37UrdSVjL8ugpweuuip1JeV1xx3Q0aG+e5Eo3Eti/fo4+KrWmjlxAh5/XC2Zdps6Ne5Aq/b+KjKFe0lUdfBt2wZnzyrcx0II8MQT8MEHqSuRRijcSySEOIs9fjx1JWMny2D8eFi7NnUl5RdC3JFu25a6EmmEwr1Eqjj4sgxWr4ZJk1JXUn5r1sQdqfruxaBwL5E1a6CrqzqtmaNH4ckn1ZIZK5MmxR1pVd5fRddQuJvZnWZ2wMwOmtk3L7PNPzWz581sn5n9qLVlSiMmTYJVq6oz+LZsiZc6VriPnRDiDvXo0dSVyHCGDXcz6wC+A9wFLALuNbNFF22zEPgjYK27Lwa+3oZapQFVGnxZBhMnwsqVqSupjhDiPXs3b05diQynkZn7CuCgux9y99PAQ8A9F23zu8B33P0ogLu/3doypVFVGnxZFtdfd3WlrqQ6Vq6MO1T13fOvkXCfBbx23veHB352vhuBG81su5ntMrM7B3sgM7vPzPrMrK+/v7+5imVI9cFX9tbMW2/Bc8/prktjrasL1q0r//urDBoJ98FuNewXfd8JLAQ2AvcC3zOzaZf8I/cH3b3H3Xu6u7tHWqs0oKsrzmbLPvg2bYpf1W8feyHEHetbb6WuRIbSSLgfBuac9/1s4Mgg2/xvdz/j7i8BB4hhLwmEAPv2lXvwZVk8cWvZstSVVE99h1rfwUo+NRLuu4GFZjbfzLqALwGPXLTN/wJqAGY2ndimOdTKQqVxVRh8WRavM97ZmbqS6lm2LO5Yy/7psOiGDXd3Pwt8BXgM2A887O77zOwBM7t7YLPHgHfN7HmgF/iX7v5uu4qWoS1dGi+iVdbB99prcPCgWjKpdHbGaxnpoGq+NTTvcfdHgUcv+tn95/3dgW8M/JHEOjvjrLas4V4PFYV7OiHAT38ad7Rz5gy/vYw9naFaUiHE2e2rr6aupPWyLN6g5OabU1dSXfUdq2bv+aVwL6myDj73GO61WrxBh6Rxyy3x9ntl/XRYBhoeJbV4MXR3l2/wvfhibAWoJZPWuHHx3r29vXGHK/mjcC+pcePi7DbLyjX46jsrhXt6IcS23yGti8slhXuJhQCHD8fee1lkGcyaBQt1FkVy9R1s2T4dloXCvcTqp+aXZfDV++0hgA123rSMqc9+FmbOLM/7q2wU7iW2cGGc5ZZl8O3bB/39asnkhVmcQKjvnk8K9xIzi0HY2xuve1509Z2ULhaWHyHEy1zs35+6ErmYwr3kQoiz3X37UlcyelkGCxbA3LmpK5E69d3zS+FecmXpu587F6+Vo5ZMvsyfD/PmFf/9VUYK95KbOxduuKH4JzPt3QvHjinc8yiEuOMtQ+uvTBTuFVAffOfOpa6keeq351etFm/r+PTTqSuR8yncKyCEOOvduzd1Jc3r7YVFi+C661JXIhcrS+uvbBTuFVD0wXf6NGzdqpZMXs2aFde8F/X9VVYK9wqYMSNea6aog2/3bjhxQuGeZyHAli1w5kzqSqRO4V4RIcTZ7+nTqSsZuSyLa/Y3bEhdiVxOrQbHj8OePakrkTqFe0WEAB9+CE88kbqSkcuyeHepT30qdSVyORs3xq9F/XRYRgr3itiwIc5+izb4Tp6EHTvUksm77m649dbivb/KTOFeEVdfHWe/RRt8O3bEVpLCPf9CgO3b4aOPUlcioHCvlBBg587YnimKLIv3hF23LnUlMpxaDU6dgl27UlcioHCvlBDiLHjHjtSVNC7LYPlymDIldSUynPXr401iivbpsKwU7hWybl2cBRdl8L3/flwGqZZMMUybBrffXpz3V9kp3CtkyhRYsaI4g2/r1njJBIV7cYQAjz8ez0uQtBTuFRMC9PXFWXHeZRlMmACrV6euRBpVq8UTmbZvT12JKNwrJoQ4G966NXUlw8syWLMGJk5MXYk0qmitvzJTuFfM6tVxNpz3wffuu/Eqg2rJFMvkybBqVf7fX1WgcK+YK66AtWvzP/g2b4735VS4F08I8TIEx46lrqTaFO4VFAI89VScHedVlsVZ4PLlqSuRkarV4o07tmxJXUm1KdwrqD4b3rQpaRlDyrK4bnr8+NSVyEitWhU/Ieb902HZKdwrqKcHrrwyv4PvjTdg/361ZIqqKK2/slO4V9D48XFWnNfBV7/fq8K9uEKAZ56B/v7UlVSXwr2iajV44QU4ciR1JZfKsnihsyVLUlcizarvmDdvTltHlSncK6o++Oqz5DzJsnh98I6O1JVIs26/Pd+tvypQuFfUkiVxdpy3wffSS/FP/b6vUkx5b/1VgcK9ojo64uw4b4NP/fbyCAEOHMhn668KFO4VFgK8/HKcKedFlsG118KiRakrkdHKc+uvChoKdzO708wOmNlBM/vmENv9EzNzM+tpXYnSLvXBl5fZu3usJYR4S0Aptry2/qpi2HA3sw7gO8BdwCLgXjO7ZF5lZlOArwGPt7pIaY+bboIZM/IzszpwIK5xV0umHMaNy2frryoambmvAA66+yF3Pw08BNwzyHZ/AnwbONXC+qSNzGKQZlmcNaemfnv55LH1VxWNhPss4LXzvj888LNfMbOlwBx3/+lQD2Rm95lZn5n19evshlwIIc6WDxxIXUncyVx/PSxYkLoSaRX13dNpJNwH637+ap5nZuOA/wD8wXAP5O4PunuPu/d0d3c3XqW0TV767h9/HANA/fZyqbf+Ur+/qqiRcD8MzDnv+9nA+YubpgA3A5vM7GVgFfCIDqoWw/z5MHdu+sH37LPxKpVqyZSLWTxnIS+tvyppJNx3AwvNbL6ZdQFfAh6p/9Ldj7n7dHef5+7zgF3A3e7e15aKpaXqfffe3jh7TqW+c9HJS+WTp9ZflQwb7u5+FvgK8BiwH3jY3feZ2QNmdne7C5T2CwHeey9e6CmVLIMbb4TZs9PVIO2Rl9Zf1TS0zt3dH3X3G939Bnf/04Gf3e/ujwyy7UbN2oulPltONfjOno0XmFJLppwWLIA5c3RQdazpDFVh1qw4a04V7nv2wAcfKNzLKi+tv6pRuAsQB9/mzXDmzNg/d32nsnHj2D+3jI0Q4gHzZ59NXUl1KNwFiIPv+PE4ix5rWQa33AJaHVteqVt/VaRwF+CTWfNYD76PPoJt29SSKbs5c2DhQvXdx5LCXYA4a7711rEP91274NQphXsV1Gqx9Xf2bOpKqkHhLr8SAmzfHmfTYyXL4gWm1q8fu+eUNEKA99+HJ59MXUk1KNzlV0KIs+hdu8buOXt74y3Zpk0bu+eUNFK1/qpK4S6/sn59nEWP1eA7cSLuSNSSqYYZM+Dmm9V3HysKd/mVq66Cnp6xC/ft2+PSS4V7ddRqsHUrnD6dupLyU7jLBUKIs+kTJ9r/XFkWb6S8dm37n0vyIQQ4eRIe1y192k7hLhcIIa5m2Lat/c+VZbBqFUye3P7nknzYsCGesaq+e/sp3OUCa9fG2XS7B98vfxlPmFJLplquvhqWLVPffSwo3OUCkybB6tXtD/ctW+J1RhTu1VOrwc6d8OGHqSspN4W7XKJWi2uRjx5t33NkGUycCCtXtu85JJ9CiAdUd+xIXUm5KdzlEiHEWfWWLe17jiyLLaAJE9r3HJJP69ZBZ6f67u2mcJdLrFwZZ9XtGnxvvx2vDqiWTDVNmQIrVqjv3m4Kd7nEhAlxdtWucN+0KX5VuFdXrQa7d8fLEUh7KNxlUCHAc8/FWXarZVmcvd1+e+sfW4ohBDh3Lp7QJO2hcJdB1WfV9Vl2K2VZXO/c2dn6x5ZiWL06fkJU3719FO4yqGXLYOrU1g++w4fhF79QS6bqJk6ENWvUd28nhbsMqrMzzq5bHe71waxwlxDgqafi7fek9RTuclkhxFn2a6+17jGzDK65Jt5WT6qtVgP3eAMPaT2Fu1xWfXbdqo/O7jHca7V4aWGptuXL43WF1HdvDw0xuaybb4bp01s3+A4dgldfVUtGoq4uuOMOhXu7KNzlssaNi7PsLIuz7tGqD2KFu9SFAPv3w5tvpq6kfBTuMqQQYs/9xRdH/1hZBp/+NNx44+gfS8qhVotftWqm9RTuMqT64BvtR+d6vz2EeD1vEYClS+MdwNSaaT2FuwzpxhvjbHu0g+/55+PZrvWdhQhAR0e8cbbCvfUU7jIkszjbHm3fXf12uZwQ4sH2V15JXUm5KNxlWCFAfz/s29f8Y2QZzJ8P8+a1rCwpCfXd20PhLsOqz7ab/eh87ly8Ro1m7TKYxYuhu1utmVZTuMuw5s6FBQuaH3xPPRXvmapwl8G0esmtRAp3aUgI8TTxc+dG/m/rH7d1MFUuJwR4/XU4eDB1JeWhcJeGhBBn3089NfJ/m2Vw000wc2br65JyaNWSW/mEwl0a0uzgO3Mm3otVLRkZysKFMGuWwr2VGgp3M7vTzA6Y2UEz++Ygv/+GmT1vZs+Y2c/NbG7rS5WUrrsOFi0a+eDbvRtOnFC4y9DqS257e9V3b5Vhw93MOoDvAHcBi4B7zWzRRZvtBXrc/VbgJ8C3W12opBdCvC3a6dON/5ssiwN3w4b21SXl0Iolt/KJRmbuK4CD7n7I3U8DDwH3nL+Bu/e6+4cD3+4CZre2TMmDEOIsfPfuxv9NlsFtt8VruIsMRX331mok3GcB59+u4fDAzy7nd4D/M5qiJJ82bIiz8EYH38mTsGOHWjLSmNEuuZULNRLug13madCumJl9GegB/uwyv7/PzPrMrK+/v7/xKiUXPvWpOAtvdPDt3AkffaQlkNK4EOIJb80suZULNRLuh4E5530/Gzhy8UZm9nng3wB3u/tHgz2Quz/o7j3u3tPd3d1MvZJYCHE2fvLk8NtmWbww1B13tL8uKYcQ4Nix5pbcyoUaCffdwEIzm29mXcCXgEfO38DMlgLfJQb7260vU/IihHhAdceO4bfNsngrtalT21+XlIP67q0zbLi7+1ngK8BjwH7gYXffZ2YPmNndA5v9GXAl8D/M7Ckze+QyDycFd8cdcTY+3OD74AN44gn122VkrrsunvCmcB+9zkY2cvdHgUcv+tn95/398y2uS3JqyhRYsWL4wbd1a+ybKtxlpEKAH/4wfkLs6kpdTXHpDFUZsRDicsgPPrj8NlkWB+aaNWNXl5RDM0tu5VIKdxmxEOKsfOvWy2/T2xuDfeLEsatLyqG+5FbXdx8dhbuM2OrVMGHC5Vsz770He/eqJSPNueYaWLJEfffRUrjLiE2cGGfllxt8mzfH64Mo3KVZI1lyK4NTuEtTQohrkd9999LfZRlMnhyXQYo0I4R4AtzOnakrKS6FuzQlhDg737z50t9lWVwyqZUO0qz6klv13ZuncJemLF8eZ+cXt2befBOef14tGRmdqVOhp0d999FQuEtTxo+H9esvHXz1mZbCXUYrhHgi3FBLbuXyFO7StFoN9u+HN9745GdZBtOmxQuMiYxGCHD2LGzblrqSYlK4S9Pqs/Pz+6JZFtcpd3SkqUnKY82aeNxGfffmKNylabfdFmfp9dbMyy/DoUNqyUhrTJoEq1ap794shbs0raMDNm78ZPCp3y6tFgI8+SQcPZq6kuJRuMuohAAvvRT/ZBl0d8PixamrkrIYasmtDE3hLqNyft89y+L3Nti9u0SasHJlPCNaffeRU7jLqCxaBNdeC9/9Lhw5opaMtFZXVzyhSX33kVO4y6iYfbIeGRTu0nq1Gjz3HLz1VupKikXhLqNWD/Q5c+CGG9LWIuVTf39t2pS0jMJp6E5MIkOpDz7126Udli2LlyP46lfhgQdSV9Ma998PX/xie59D4S6jtmAB/MmfwG/8RupKpIw6O+Hb34a///vUlbTO1Ve3/znM3dv/LIPo6enxvr6+JM8tIlJUZrbH3XuG2049dxGRElK4i4iUkMJdRKSEFO4iIiWkcBcRKSGFu4hICSncRURKSOEuIlJCyU5iMrN+4JUm//l04J0WlpOSXkv+lOV1gF5LXo3mtcx19+7hNkoW7qNhZn2NnKFVBHot+VOW1wF6LXk1Fq9FbRkRkRJSuIuIlFBRw/3B1AW0kF5L/pTldYBeS161/bUUsucuIiJDK+rMXUREhlC4cDezO83sgJkdNLNvpq6nWWb2AzN728yeS13LaJjZHDPrNbP9ZrbPzH4/dU3NMrMrzOwJM3t64LX8u9Q1jZaZdZjZXjP7aepaRsPMXjazZ83sKTMr7I0gzGyamf3EzF4YGDOr2/ZcRWrLmFkH8P+AfwAcBnYD97r780kLa4KZrQeOA3/l7jenrqdZZjYTmOnuT5rZFGAP8I8L+v/EgMnuftzMxgPbgN93912JS2uamX0D6AGmuvuvp66nWWb2MtDj7oVe525mfwlsdffvmVkXMMndf9mO5yrazH0FcNDdD7n7aeAh4J7ENTXF3bcA76WuY7Tc/Q13f3Lg7x8A+4FZaatqjkfHB74dP/CnOLOfi5jZbOAfAd9LXYuAmU0F1gPfB3D30+0KdiheuM8CXjvv+8MUNEjKyMzmAUuBx9NW0ryBNsZTwNvAz9y9sK8F+I/AvwI+Tl1ICzjwd2a2x8zuS11MkxYA/cB/HWiVfc/MJrfryYoW7jbIzwo7syoTM7sS+Bvg6+7+fup6muXu59z9NmA2sMLMCtkyM7NfB9529z2pa2mRte6+DLgL+L2BtmbRdALLgL9w96XACaBtxw2LFu6HgTnnfT8bOJKoFhkw0J/+G+Cv3f1/pq6nFQY+Lm8C7kxcSrPWAncP9KofAoKZ/fe0JTXP3Y8MfH0b+FswEDb5AAABE0lEQVRii7ZoDgOHz/s0+BNi2LdF0cJ9N7DQzOYPHIz4EvBI4poqbeAg5PeB/e7+71PXMxpm1m1m0wb+PhH4PPBC2qqa4+5/5O6z3X0ecZxk7v7lxGU1xcwmDxysZ6CN8Q+Bwq0yc/c3gdfM7LMDP/o1oG0LDzrb9cDt4O5nzewrwGNAB/ADd9+XuKymmNmPgY3AdDM7DPyxu38/bVVNWQv8M+DZgV41wL9290cT1tSsmcBfDqzKGgc87O6FXkJYEjOAv43zCDqBH7n7/01bUtO+Cvz1wOT0EPDb7XqiQi2FFBGRxhStLSMiIg1QuIuIlJDCXUSkhBTuIiIlpHAXESkhhbuISAkp3EVESkjhLiJSQv8f5NS8u7riyf4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"隐层2的节点数目对于准确率的影响\")\n",
    "pyplot.plot(acc_array.keys(), acc_array.values(), 'b-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "添加第三个隐层好像没没啥意义了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "#包装函数方便调优\n",
    "\n",
    "def train_withLearningRate_L2(layer_nodes = [500,200], use_l2 = False, l2_rate = 0.0001):\n",
    "    # Create the model\n",
    "    x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "\n",
    "    \n",
    "    y = x\n",
    "    current_node = 784\n",
    "    WW = []\n",
    "    for i in range(len(layer_nodes)):\n",
    "        node_count = layer_nodes[i]\n",
    "        print(str(current_node) + \"_\" + str(node_count))\n",
    "        W = tf.Variable(tf.truncated_normal([current_node, node_count],stddev = 0.1))\n",
    "        b = tf.Variable(tf.constant(0.1,shape = [node_count]))\n",
    "        y = tf.nn.relu(tf.matmul(y, W) + b)\n",
    "        current_node = node_count\n",
    "        WW.append(W)\n",
    "    W = tf.Variable(tf.truncated_normal([current_node, 10],stddev = 0.1))\n",
    "    WW.append(W)\n",
    "    b = tf.Variable(tf.constant(0.1,shape = [10]))\n",
    "    y = tf.matmul(y, W) + b\n",
    "    \n",
    "   \n",
    "    y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "    # The raw formulation of cross-entropy,\n",
    "    #\n",
    "    #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "    #                                 reduction_indices=[1]))\n",
    "    #\n",
    "    # can be numerically unstable.\n",
    "    #\n",
    "    # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "    # outputs of 'y', and then average across the batch.\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    "    )\n",
    "    loss = cross_entropy\n",
    "    \n",
    "    if use_l2:\n",
    "        #计算l2正则化损失函数\n",
    "        regularizer = tf.contrib.layers.l2_regularizer(l2_rate)\n",
    "        #计算模型的正则化损失，只计算权重，不使用b\n",
    "        regularization = 0\n",
    "        for w in WW:\n",
    "            regularization =regularization + regularizer(w)\n",
    "        loss = loss + regularization\n",
    "    \n",
    "    batchSize = 100\n",
    "    trainingStep = 10000\n",
    "    Learning_rate_base = 0.5\n",
    "    Learning_rate_decay = 0.99\n",
    "    global_step = tf.Variable(0,trainable = False)\n",
    "    \n",
    "    Learning_rate = tf.train.exponential_decay(\n",
    "        Learning_rate_base,\n",
    "        global_step,\n",
    "        mnist.train.num_examples / batchSize,\n",
    "        Learning_rate_decay)\n",
    "    \n",
    "    train_step = tf.train.GradientDescentOptimizer(Learning_rate).minimize(loss,global_step = global_step)\n",
    "\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    # Train\n",
    "    for _ in range(trainingStep):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batchSize)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        \n",
    "        if _ %1000 == 0:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            acc = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "            print(\"batch \" + str(_) + \" acc is \" + str(acc))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    ret = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "    print(ret)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.2553\n",
      "batch 500 acc is 0.9593\n",
      "batch 1000 acc is 0.9709\n",
      "batch 1500 acc is 0.9772\n",
      "batch 2000 acc is 0.9783\n",
      "batch 2500 acc is 0.9788\n",
      "batch 3000 acc is 0.9806\n",
      "batch 3500 acc is 0.9808\n",
      "batch 4000 acc is 0.981\n",
      "batch 4500 acc is 0.9822\n",
      "batch 5000 acc is 0.9835\n",
      "batch 5500 acc is 0.9825\n",
      "batch 6000 acc is 0.9824\n",
      "batch 6500 acc is 0.9831\n",
      "batch 7000 acc is 0.9824\n",
      "batch 7500 acc is 0.9827\n",
      "batch 8000 acc is 0.9829\n",
      "batch 8500 acc is 0.9829\n",
      "batch 9000 acc is 0.9828\n",
      "batch 9500 acc is 0.9826\n",
      "batch 10000 acc is 0.9827\n",
      "batch 10500 acc is 0.9827\n",
      "batch 11000 acc is 0.9829\n",
      "batch 11500 acc is 0.9827\n",
      "batch 12000 acc is 0.9824\n",
      "batch 12500 acc is 0.9825\n",
      "batch 13000 acc is 0.9826\n",
      "batch 13500 acc is 0.9828\n",
      "batch 14000 acc is 0.9825\n",
      "batch 14500 acc is 0.9827\n",
      "batch 15000 acc is 0.9828\n",
      "batch 15500 acc is 0.9827\n",
      "batch 16000 acc is 0.9827\n",
      "batch 16500 acc is 0.9825\n",
      "batch 17000 acc is 0.9828\n",
      "batch 17500 acc is 0.9827\n",
      "batch 18000 acc is 0.9824\n",
      "batch 18500 acc is 0.9828\n",
      "batch 19000 acc is 0.9826\n",
      "batch 19500 acc is 0.9828\n",
      "batch 20000 acc is 0.9826\n",
      "batch 20500 acc is 0.983\n",
      "batch 21000 acc is 0.9827\n",
      "batch 21500 acc is 0.9827\n",
      "batch 22000 acc is 0.9826\n",
      "batch 22500 acc is 0.9829\n",
      "batch 23000 acc is 0.9828\n",
      "batch 23500 acc is 0.9828\n",
      "batch 24000 acc is 0.9827\n",
      "batch 24500 acc is 0.9828\n",
      "batch 25000 acc is 0.9825\n",
      "batch 25500 acc is 0.9827\n",
      "batch 26000 acc is 0.9825\n",
      "batch 26500 acc is 0.9828\n",
      "batch 27000 acc is 0.9827\n",
      "batch 27500 acc is 0.9829\n",
      "batch 28000 acc is 0.9826\n",
      "batch 28500 acc is 0.9828\n",
      "batch 29000 acc is 0.9827\n",
      "batch 29500 acc is 0.9827\n",
      "0.9826\n"
     ]
    }
   ],
   "source": [
    "_layer_nodes = [800,160] #[600,500,400,10]\n",
    "accuraccy = train_withLearningRate_L2(layer_nodes = _layer_nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5000次的时候达到峰值0.9835，后面由于学习率衰减，逐渐收敛到0.9827"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.0974\n",
      "batch 500 acc is 0.965\n",
      "batch 1000 acc is 0.968\n",
      "batch 1500 acc is 0.9708\n",
      "batch 2000 acc is 0.9735\n",
      "batch 2500 acc is 0.9775\n",
      "batch 3000 acc is 0.9788\n",
      "batch 3500 acc is 0.9805\n",
      "batch 4000 acc is 0.9798\n",
      "batch 4500 acc is 0.9814\n",
      "batch 5000 acc is 0.9823\n",
      "batch 5500 acc is 0.982\n",
      "batch 6000 acc is 0.9823\n",
      "batch 6500 acc is 0.9827\n",
      "batch 7000 acc is 0.983\n",
      "batch 7500 acc is 0.9825\n",
      "batch 8000 acc is 0.9819\n",
      "batch 8500 acc is 0.9822\n",
      "batch 9000 acc is 0.9826\n",
      "batch 9500 acc is 0.9825\n",
      "batch 10000 acc is 0.9821\n",
      "batch 10500 acc is 0.9826\n",
      "batch 11000 acc is 0.9819\n",
      "batch 11500 acc is 0.9819\n",
      "batch 12000 acc is 0.9818\n",
      "batch 12500 acc is 0.9818\n",
      "batch 13000 acc is 0.9824\n",
      "batch 13500 acc is 0.9815\n",
      "batch 14000 acc is 0.9821\n",
      "batch 14500 acc is 0.9824\n",
      "batch 15000 acc is 0.982\n",
      "batch 15500 acc is 0.9822\n",
      "batch 16000 acc is 0.9822\n",
      "batch 16500 acc is 0.9825\n",
      "batch 17000 acc is 0.9825\n",
      "batch 17500 acc is 0.982\n",
      "batch 18000 acc is 0.9826\n",
      "batch 18500 acc is 0.9812\n",
      "batch 19000 acc is 0.982\n",
      "batch 19500 acc is 0.9818\n",
      "batch 20000 acc is 0.9819\n",
      "batch 20500 acc is 0.9824\n",
      "batch 21000 acc is 0.9816\n",
      "batch 21500 acc is 0.9821\n",
      "batch 22000 acc is 0.9827\n",
      "batch 22500 acc is 0.9821\n",
      "batch 23000 acc is 0.9831\n",
      "batch 23500 acc is 0.9824\n",
      "batch 24000 acc is 0.982\n",
      "batch 24500 acc is 0.982\n",
      "batch 25000 acc is 0.9826\n",
      "batch 25500 acc is 0.9817\n",
      "batch 26000 acc is 0.9824\n",
      "batch 26500 acc is 0.9825\n",
      "batch 27000 acc is 0.9823\n",
      "batch 27500 acc is 0.9822\n",
      "batch 28000 acc is 0.9821\n",
      "batch 28500 acc is 0.9827\n",
      "batch 29000 acc is 0.9821\n",
      "batch 29500 acc is 0.9827\n",
      "0.9826\n"
     ]
    }
   ],
   "source": [
    "_layer_nodes = [800,160]\n",
    "accuraccy = train_withLearningRate_L2(layer_nodes = _layer_nodes, use_l2 = True, l2_rate = 0.0001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "l2调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1281\n",
      "batch 1000 acc is 0.5178\n",
      "batch 2000 acc is 0.6413\n",
      "batch 3000 acc is 0.5528\n",
      "batch 4000 acc is 0.7266\n",
      "batch 5000 acc is 0.4185\n",
      "batch 6000 acc is 0.687\n",
      "batch 7000 acc is 0.5161\n",
      "batch 8000 acc is 0.5256\n",
      "batch 9000 acc is 0.5937\n",
      "0.6542\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1978\n",
      "batch 1000 acc is 0.9416\n",
      "batch 2000 acc is 0.9105\n",
      "batch 3000 acc is 0.931\n",
      "batch 4000 acc is 0.9355\n",
      "batch 5000 acc is 0.9416\n",
      "batch 6000 acc is 0.9326\n",
      "batch 7000 acc is 0.9404\n",
      "batch 8000 acc is 0.9356\n",
      "batch 9000 acc is 0.9478\n",
      "0.9424\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1862\n",
      "batch 1000 acc is 0.969\n",
      "batch 2000 acc is 0.9758\n",
      "batch 3000 acc is 0.9768\n",
      "batch 4000 acc is 0.9784\n",
      "batch 5000 acc is 0.9666\n",
      "batch 6000 acc is 0.9815\n",
      "batch 7000 acc is 0.9803\n",
      "batch 8000 acc is 0.9766\n",
      "batch 9000 acc is 0.98\n",
      "0.9809\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.178\n",
      "batch 1000 acc is 0.9742\n",
      "batch 2000 acc is 0.9723\n",
      "batch 3000 acc is 0.9764\n",
      "batch 4000 acc is 0.9809\n",
      "batch 5000 acc is 0.9823\n",
      "batch 6000 acc is 0.9824\n",
      "batch 7000 acc is 0.9822\n",
      "batch 8000 acc is 0.9825\n",
      "batch 9000 acc is 0.982\n",
      "0.9825\n",
      "评分最好的l2_rate和acc是：0.0001,0.9825\n"
     ]
    }
   ],
   "source": [
    "acc_array = {}\n",
    "\n",
    "best_j = 0\n",
    "best_acc = 0\n",
    "l2_rates = [0.1,0.01,0.001,0.0001]\n",
    "for j in range(len(l2_rates)):\n",
    "    l2_rate = l2_rates[j]\n",
    "    _layer_nodes = [800,160] #[600,500,400,10]\n",
    "    accuraccy = train_withLearningRate_L2(layer_nodes = _layer_nodes,use_l2 = True, l2_rate = l2_rate)\n",
    "    acc_array[j] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_j = j\n",
    "    \n",
    "print(\"评分最好的l2_rate和acc是：\"+ str(l2_rates[best_j]) + \",\" + str(best_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l2_rate大小对于准确率的影响\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x4c4c7748>]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHlNJREFUeJzt3Xt81dWV9/HPMohgVURI1XIL+sILXvGJ18xTfeqoVKcyatsBx1Fap0xbQYuoBVFBbBW13mixQisKrRYZqzUq1QdHHDsFlSB4IRZEHCEGxwgqVFEgrPljnzSHkJhfyMnZ5/J9v17nlXN+v32StTlknZX922dvc3dERKQ47BI7ABERyR4lfRGRIqKkLyJSRJT0RUSKiJK+iEgRUdIXESkiSvoiIkVESV9EpIgo6YuIFJFOsQNoqmfPnl5WVhY7DBGRvLJ48eIP3L20tXY5l/TLysqoqqqKHYaISF4xs3eStNPwjohIEVHSFxEpIkr6IiJFRElfRKSIKOmLiBQRJX0RkSKipC8iUkRybp6+iEhr3MOtvh62bdvxa3PHculcS+179YIRIzr2305JX6SI/M//wJ//DIsXw6ZN8ZPczp4r1K29TzhBSV9EdlJ9PVRXhyS/YEH4umpVOFdSAl27hq+77NL4Nf3+zp7r3Dnz37MYzpmFW0dT0hcpEBs3wksvNSb5hQthw4Zw7stfhooK+OEP4aST4JhjYLfd4sYrcSjpi+Qhd1i9urGCX7AAXnklDH+YweGHw7BhIdGfdBIccEB2qkjJfUr6InlgyxZYunT7JP/uu+Hcl74UxoLHjw9J/vjjYe+948YruUtJXyQHrV8fhmcakvxLL4ULrwB9+8JXvxoq+IoKOOII6KTfZElI/1VEInOHN9/c/oLrG2+EcyUlMGhQmNFx0knh1rt33Hglvynpi2TZpk1hymRDkl+wAD74IJzr3h1OPBEuuCAk+GOPDcM3IpmipC/Swd57b/ux+MWLwxg9wEEHwTe+0VjFH3JImMIn0lGU9EUyqL4eli3bfqjm7bfDud12C5X75ZeHBH/iiVDa6uZ2IpmlpC/SDhs3wosvNib5F15onBu/777hQuvIkY1z4zt3jhuviJK+SELu8M472w/VvPpq49z4I46A889vnBvfv7/mxkvuUdIXacGWLbBkyfZJvrY2nNtjjzA3/pprGufGd+sWN16RJJT0RVLWr2+cTfPnP8OiRY1z4/v1g1NOaZwbf/jhmhsv+Un/baUoucOKFdtfcP3LX8K5Tp3C3Ph/+7fGWTW9esWNVyRTlPSlKGzaBFVV28+NX7cunOvePST2Cy9snBu/++5x4xXpKImSvpkNBu4CSoBfu/vkJuf7ATOAUmA9cIG716TO1QOvpZqudvezMxS7SIvWrt1+LP7llxvnxh98MJx9duMF14MP1tx4KR6tJn0zKwGmAqcBNcAiM6t09+q0Zj8DZrn7TDP7GnAT8C+pc5vc/egMxy3yN/X18Prr2yf5hrnxXbo0zo2vqAhz43v2jBuvSExJKv3jgJXuvgrAzGYDQ4D0pD8QGJ26Px/4QyaDFEm3cWOYD9+Q5F94IRwD2G+/xrnxFRVhbF5z40UaJUn6vYA1aY9rgOObtHkFOI8wBHQOsKeZ9XD3dUAXM6sCtgKT3V1vCJJYw9z49Auur73WODf+yCMb16mpqICyMs2NF/kiSZJ+c79CTXeovAL4hZkNB54H3iUkeYC+7l5rZgcAz5rZa+7+1nY/wGwEMAKgb9++bQhfCs3mzWHd+PQkv3ZtONcwN/7aaxvnxu+1V9x4RfJNkqRfA/RJe9wbqE1v4O61wLkAZrYHcJ67f5x2DndfZWbPAYOAt5o8fzowHaC8vLxAtzyW5qxb1zibZsGCsG78Z5+Fc2Vl8LWvbT83vqQkargieS9J0l8EDDCz/oQKfihwfnoDM+sJrHf3bcA4wkwezKw78Km7f55qUwHcksH4JY+4w/Ll21fxy5eHc506hbVpfvCDxrnxX/lK3HhFClGrSd/dt5rZSOBpwpTNGe6+zMwmAVXuXgmcAtxkZk4Y3rkk9fRDgWlmtg3YhTCmX73DD5GCtXYt3H9/YyW/fn04vs8+IbEPH944N75r15iRihQHc8+t0ZTy8nKvqqqKHYZkQH19SOZLloR14huGaRrmxuuCq0jmmNlidy9vrZ0+kSsd5t57Q8J/8EEYNix2NCICYchFJOM+/BCuvjps4D10aOxoRKSBkr50iIkTQ+KfMkXDOCK5RElfMu7112Hq1LBK5VFHxY5GRNIp6UtGucNll4UPTd1wQ+xoRKQpXciVjHr0UXj22VDp9+gROxoRaUqVvmTMpk1hNcsjjoARI2JHIyLNUaUvGXPrrWFxtPnztZWgSK5SpS8ZsXo1TJ4M3/522EtWRHKTkr5kxJVXhq+33ho3DhH5Ykr60m7PPQdz5sDYsaCVsUVym5K+tMvWrWGKZr9+jdW+iOQuXW6Tdpk+HV59FR5+WKtkiuQDVfqy09atg2uuCRudnHtu7GhEJAklfdlp110HGzbAXXdpfR2RfKGkLzvllVfgnnvghz8M2xiKSH5Q0pc2c4dLL4Xu3eH662NHIyJtoQu50mZz5sDzz8O0aSHxi0j+UKUvbfLJJ2Fq5qBBcPHFsaMRkbZSpS9tcvPNsGZN2AKxpCR2NCLSVqr0JbG334ZbboHzz4e/+7vY0YjIzkiU9M1ssJktN7OVZja2mfP9zOw/zOxVM3vOzHqnnbvIzN5M3S7KZPCSXVdcEar7m2+OHYmI7KxWk76ZlQBTga8DA4FhZjawSbOfAbPc/UhgEnBT6rn7ABOA44HjgAlmpkt/eeiZZ+CRR2D8eOjdu/X2IpKbklT6xwEr3X2Vu28GZgNDmrQZCPxH6v78tPNnAPPcfb27fwjMAwa3P2zJpi1bwvo6BxwQNkkRkfyVJOn3AtakPa5JHUv3CnBe6v45wJ5m1iPhcyXH/fKXUF0Nt98OXbrEjkZE2iNJ0m/uA/be5PEVwMlmtgQ4GXgX2JrwuZjZCDOrMrOqurq6BCFJttTVheUWTj8dzj47djQi0l5Jkn4N0CftcW+gNr2Bu9e6+7nuPggYnzr2cZLnptpOd/dydy8vLS1tYxekI40fH+bm33mn1tcRKQRJkv4iYICZ9TezzsBQoDK9gZn1NLOG7zUOmJG6/zRwupl1T13APT11TPLAyy/Dr38No0bBoYfGjkZEMqHVpO/uW4GRhGT9BjDH3ZeZ2SQza/iD/xRguZmtAPYFfpp67nrgBsIbxyJgUuqY5Dj3kOxLS2HChNjRiEimJPpErrvPBeY2OXZd2v2HgYdbeO4MGit/yRMPPggLFsC990K3brGjEZFM0SdyZQd//StcdRWUl8Pw4bGjEZFM0to7soMbb4Ta2rAF4i4qC0QKin6lZTsrV8Jtt8GFF8KJJ8aORkQyTUlftjNmDHTuDJMnx45ERDqChnfkb556Ciorw4Jq++8fOxoR6Qiq9AWAzZvhRz+CAQPCOjsiUphU6QsAv/gFLF8OTzwBu+0WOxoR6Siq9IX33oOJE+HMM+Gss2JHIyIdSUlfuPpq+OwzuOOO2JGISEdT0i9yL70E990XxvMPOih2NCLS0ZT0i9i2bXDppbDffnDNNbGjEZFs0IXcIvab38CLL8LMmbDXXrGjEZFsUKVfpDZsgLFj4fjj4YILYkcjItmiSr9I/eQnYdZOZaXW1xEpJvp1L0IrVoSdsL77XTj22NjRiEg2KekXodGjoWvXsJqmiBQXDe8UmSefhLlzw0qa++4bOxoRyTZV+kXk88/DfPxDDoGRI2NHIyIxqNIvInfdFdbLf+qpsHyyiBQfVfpForYWbrgBzj4bzjgjdjQiEouSfpEYOzYsn3z77bEjEZGYEiV9MxtsZsvNbKWZjW3mfF8zm29mS8zsVTM7M3W8zMw2mdnS1O2eTHdAWrdwYfj07ZgxcOCBsaMRkZhaHdM3sxJgKnAaUAMsMrNKd69Oa3YNMMfdf2lmA4G5QFnq3FvufnRmw5aktm2DUaPgK18Jq2mKSHFLciH3OGClu68CMLPZwBAgPek70LB6SzegNpNBys677z5YvBgeeAD22CN2NCISW5LhnV7AmrTHNalj6SYCF5hZDaHKH5V2rn9q2Oc/zez/tidYaZuPPoJx46CiAoYNix2NiOSCJEnfmjnmTR4PA+53997AmcBvzGwXYC3Q190HAZcDD5rZDus5mtkIM6sys6q6urq29UBaNGkSfPABTJkC1tyrKCJFJ0nSrwH6pD3uzY7DNxcDcwDcfSHQBejp7p+7+7rU8cXAW8AOW3W4+3R3L3f38tLS0rb3QnZQXQ0//zl873twzDGxoxGRXJEk6S8CBphZfzPrDAwFKpu0WQ2cCmBmhxKSfp2ZlaYuBGNmBwADgFWZCl6a5x4+efulL4XVNEVEGrR6Idfdt5rZSOBpoASY4e7LzGwSUOXulcAY4FdmNpow9DPc3d3MvgpMMrOtQD3wfXdf32G9ESAslzxvXvgErv5wEpF05t50eD6u8vJyr6qqih1G3vrsMxg4EHbfHZYsgV13jR2RiGSDmS129/LW2mntnQJz223w9tvwzDNK+CKyIy3DUEBqasIa+eeeC6eeGjsaEclFSvoF5Kqrwidwb7stdiQikquU9AvEn/4Ev/sdXHkllJXFjkZEcpWSfgGor4dLL4U+fcJqmiIiLdGF3ALw61/D0qXw0ENh1o6ISEtU6ee5Dz+E8ePh5JPhW9+KHY2I5Dol/Tw3YUJI/HfdpfV1RKR1Svp57LXX4O674fvfh6OOih2NiOQDJf085Q6XXQZ77RVW0xQRSUIXcvPUI4/A/PkwdSr06BE7GhHJF6r089Cnn8Lll8ORR8KIEbGjEZF8oko/D916K6xeDbNmQSe9giLSBqr088w778DkyfDtb4dpmiIibaGkn2euvDJMzbz11tiRiEg+UtLPI/Pnw7//e1hqoW/f2NGISD5S0s8TW7eGKZr9+oVqX0RkZ+gyYJ6YNi18GOv3v4euXWNHIyL5SpV+Hli3Dq69NmyMcs45saMRkXympJ8Hrr0WNmzQ+joi0n5K+jlu6dIwtHPJJXDYYbGjEZF8lyjpm9lgM1tuZivNbIdtOsysr5nNN7MlZvaqmZ2Zdm5c6nnLzeyMTAZf6NzD5ij77AMTJ8aORkQKQasXcs2sBJgKnAbUAIvMrNLdq9OaXQPMcfdfmtlAYC5Qlro/FDgM+ArwjJkd5O71me5IIZozJ2yDOG0adO8eOxoRKQRJKv3jgJXuvsrdNwOzgSFN2jiwV+p+N6A2dX8IMNvdP3f3t4GVqe8nrfjkE7jiChg0CC6+OHY0IlIokkzZ7AWsSXtcAxzfpM1E4P+b2SjgS8Dfpz33hSbP7bVTkRaZyZOhpgZmz4aSktjRiEihSFLpNzdfxJs8Hgbc7+69gTOB35jZLgmfi5mNMLMqM6uqq6tLEFJhe/vtsMzC+edDRUXsaESkkCRJ+jVAn7THvWkcvmlwMTAHwN0XAl2Angmfi7tPd/dydy8vLS1NHn2BGjMmrJ55yy2xIxGRQpMk6S8CBphZfzPrTLgwW9mkzWrgVAAzO5SQ9OtS7Yaa2W5m1h8YALyUqeAL0bx58OijYbPzXhoIE5EMa3VM3923mtlI4GmgBJjh7svMbBJQ5e6VwBjgV2Y2mjB8M9zdHVhmZnOAamArcIlm7rRsy5awvs4BB8Do0bGjEZFClGjtHXefS5iGmX7surT71UCzo8/u/lPgp+2IsWjcfTe88QY89hh06RI7GhEpRPpEbo54/32YMAHOOAO+8Y3Y0YhIoVLSzxHjx4e5+XfeqfV1RKTjKOnngMWL4d57w5ILhxwSOxoRKWRK+pG5w6hRUFoK113XensRkfbQJiqRPfAALFwYKv1u3WJHIyKFTpV+RBs3wlVXQXk5DB8eOxoRKQaq9CO68UZYuxYeeQR20duviGSBUk0kK1fC7bfDhRfCCSfEjkZEioWSfiSXXw6dO4fVNEVEskXDOxH88Y/w+ONhQbX9948djYgUE1X6WbZ5M/zoRzBgQFhnR0Qkm1TpZ9nPfw4rVsCTT4bhHRGRbFKln0XvvQfXXw9nnQVnntl6exGRTFPSz6Jx4+Czz+COO2JHIiLFSkk/S156Ce6/P6yTP2BA7GhEpFgp6WfBtm1hfZ399oNrrokdjYgUM13IzYJZs0KlP3Mm7Lln7GhEpJip0u9gGzbA2LHhU7cXXBA7GhEpdqr0O9gNN4RdsR5/XOvriEh8SkMdaPnysBPWd74Dxx4bOxoRESX9DuMePnm7++5hNU0RkVyg4Z0O8uST8NRTYSXNffeNHY2ISJCo0jezwWa23MxWmtnYZs7fYWZLU7cVZvZR2rn6tHOVmQw+V33+eajyDzkERo6MHY2ISKNWK30zKwGmAqcBNcAiM6t09+qGNu4+Oq39KGBQ2rfY5O5HZy7k3HfnnfDWW/D007DrrrGjERFplKTSPw5Y6e6r3H0zMBsY8gXthwG/y0Rw+ai2NszYGTIETj89djQiIttLkvR7AWvSHtekju3AzPoB/YFn0w53MbMqM3vBzP6xheeNSLWpqqurSxh6bvrxj2HLFrjtttiRiIjsKEnSt2aOeQtthwIPu3t92rG+7l4OnA/caWYH7vDN3Ke7e7m7l5eWliYIKTctWAC//S1ccQUcuEMvRUTiS5L0a4A+aY97A7UttB1Kk6Edd69NfV0FPMf24/0Fo74eLr0UevUKq2mKiOSiJEl/ETDAzPqbWWdCYt9hFo6ZHQx0BxamHetuZrul7vcEKoDqps8tBPfdB4sXhy0Q99gjdjQiIs1rdfaOu281s5HA00AJMMPdl5nZJKDK3RveAIYBs909fejnUGCamW0jvMFMTp/1Uyg++giuvhoqKmDYsNjRiIi0LNGHs9x9LjC3ybHrmjye2MzzFgBHtCO+vHD99fDBB2GKpjV3BUREJEdoGYZ2qq4O+95+73swqCCvVohIIVHSbwd3uOyysEb+T34SOxoRkdZp7Z12eOwxeOYZmDIF8nimqYgUEVX6O2nTprDf7WGHwQ9+EDsaEZFkVOnvpNtug//+71Dpd9K/oojkCVX6O2HNGrjpJjjvPDj11NjRiIgkp6S/E666CrZtg5/9LHYkIiJto6TfRs8/D7Nnh8RfVhY7GhGRtlHSb4OG9XX69AmraYqI5BtdgmyDX/0KXnkFHnoo7H0rIpJvVOkntH49jB8PJ58M3/pW7GhERHaOkn5CEyaEhdWmTNH6OiKSv5T0E3jtNbj7bvj+9+HII2NHIyKy85T0W+EeLt7uvTdMmhQ7GhGR9tGF3Fb8/vfw3HOh0u/RI3Y0IiLto0r/C3z6KYwZE4Z0RoyIHY2ISPup0v8Ct9wCq1fDrFlQUhI7GhGR9lOl34J33oGbb4Z/+qcwTVNEpBAo6bfgiivC1Mxbb40diYhI5ijpN2P+fHj4YRg3Liy5ICJSKJT0m9i6NUzRLCsL1b6ISCFJlPTNbLCZLTezlWY2tpnzd5jZ0tRthZl9lHbuIjN7M3W7KJPBd4R77oHXXw+bpHTtGjsaEZHManX2jpmVAFOB04AaYJGZVbp7dUMbdx+d1n4UMCh1fx9gAlAOOLA49dwPM9qLDPngA7j22rAxyjnnxI5GRCTzklT6xwEr3X2Vu28GZgNDvqD9MOB3qftnAPPcfX0q0c8DBrcn4I507bWwcSPcdZfW1xGRwpQk6fcC1qQ9rkkd24GZ9QP6A8+25blmNsLMqsysqq6uLkncGbdkCUybBpdcEjY7FxEpREmSfnM1r7fQdijwsLvXt+W57j7d3cvdvby0tDRBSJnVsL5Ojx4wcWLWf7yISNYkSfo1QPrExd5AbQtth9I4tNPW50bz0EPwX/8FN94I3bvHjkZEpOMkSfqLgAFm1t/MOhMSe2XTRmZ2MNAdWJh2+GngdDPrbmbdgdNTx3LGJ5+EqZmDBsF3vxs7GhGRjtXq7B1332pmIwnJugSY4e7LzGwSUOXuDW8Aw4DZ7u5pz11vZjcQ3jgAJrn7+sx2oX1uugnefTdU+1pfR0QKnaXl6JxQXl7uVVVVWflZq1bBwIHwzW/Cb3+blR8pItIhzGyxu5e31q6oP5E7Zgx06hQWVhMRKQZFu7TyvHnwhz+Ei7e9mp2AKiJSeIqy0t+yBS67DA48EEaPbr29iEihKMpKf+pUeOMNqKyELl1iRyMikj1FV+m//z5MmABnnAH/8A+xoxERya6iS/pXXx32vr3zTq2vIyLFp6iSflUVzJgRxvMPOSR2NCIi2Vc0SX/btrC+TmlpWE1TRKQYFc2F3AcegIULQ6XfrVvsaERE4iiKSn/jRvjxj+HYY+GinN+7S0Sk4xRFpf/Tn8LatfDoo7BLUbzNiYg0r+BT4Jtvwu23hwr/+ONjRyMiElfBJ/3LLw8fwLrpptiRiIjEV9DDO3PnwhNPwC23wP77x45GRCS+gq30N28O6+ocdFCYly8iIgVc6U+ZAitWhGq/c+fY0YiI5IaCrPTXroXrr4ezzoKvfz12NCIiuaMgk/64cfD553DHHbEjERHJLQWX9F98EWbODLN2BgyIHY2ISG4pqKS/bRuMGhVm6owfHzsaEZHcU1AXcmfOhEWLYNYs2HPP2NGIiOSeRJW+mQ02s+VmttLMxrbQ5ttmVm1my8zswbTj9Wa2NHWrzFTgTX38cRjLP/FE+Od/7qifIiKS31qt9M2sBJgKnAbUAIvMrNLdq9PaDADGARXu/qGZfTntW2xy96MzHPcONm0KCX/8eK2vIyLSkiTDO8cBK919FYCZzQaGANVpbb4HTHX3DwHc/f1MB9qa/fYLC6qJiEjLktTEvYA1aY9rUsfSHQQcZGZ/NrMXzGxw2rkuZlaVOv6Pzf0AMxuRalNVV1fXpg6IiEhySSr95naS9Wa+zwDgFKA38CczO9zdPwL6unutmR0APGtmr7n7W9t9M/fpwHSA8vLypt9bREQyJEmlXwP0SXvcG6htps1j7r7F3d8GlhPeBHD32tTXVcBzwKB2xiwiIjspSdJfBAwws/5m1hkYCjSdhfMH4P8BmFlPwnDPKjPrbma7pR2vYPtrASIikkWtDu+4+1YzGwk8DZQAM9x9mZlNAqrcvTJ17nQzqwbqgSvdfZ2ZnQRMM7NthDeYyemzfkREJLvMPbeG0MvLy72qqip2GCIiecXMFrt7eWvtNKNdRKSIKOmLiBSRnBveMbM64J12fIuewAcZCiemQukHqC+5qlD6Uij9gPb1pZ+7l7bWKOeSfnuZWVWSca1cVyj9APUlVxVKXwqlH5Cdvmh4R0SkiCjpi4gUkUJM+tNjB5AhhdIPUF9yVaH0pVD6AVnoS8GN6YuISMsKsdIXEZEW5GXSb20nLzPbzcweSp1/0czKsh9lMgn6MtzM6tJ2H/vXGHG2xsxmmNn7ZvZ6C+fNzKak+vmqmR2T7RiTStCXU8zs47TX5Lpsx5iEmfUxs/lm9kZqR7vLmmmTF69Lwr7ky+vSxcxeMrNXUn25vpk2HZfD3D2vboT1f94CDgA6A68AA5u0+SFwT+r+UOCh2HG3oy/DgV/EjjVBX74KHAO83sL5M4E/EpbqPgF4MXbM7ejLKcATseNM0I/9gWNS9/cEVjTz/ysvXpeEfcmX18WAPVL3dwVeBE5o0qbDclg+Vvp/28nL3TcDDTt5pRsCzEzdfxg41cya2xcgtiR9yQvu/jyw/guaDAFmefACsLeZ7Z+d6NomQV/ygruvdfeXU/c3Am+w4wZIefG6JOxLXkj9W/819XDX1K3pxdUOy2H5mPST7OT1tzbuvhX4GOiRlejaJklfAM5L/en9sJn1aeZ8Pkja13xxYurP8z+a2WGxg2lNanhgEKGqTJd3r8sX9AXy5HUxsxIzWwq8D8xz9xZfl0znsHxM+kl28krSJhckifNxoMzdjwSeofHdP9/ky2uSxMuEj7wfBfycsJ9EzjKzPYDfAz9y9w1NTzfzlJx9XVrpS968Lu5e7+5HEzalOs7MDm/SpMNel3xM+kl38uoDYGadgG7k5p/rrfbF3de5++eph78C/k+WYsu0JK9bXnD3DQ1/nrv7XGDX1CZBOcfMdiUkyQfc/ZFmmuTN69JaX/LpdWngYUvZ54DBTU51WA7Lx6SfZCevSuCi1P1vAs966opIjmm1L03GV88mjGXmo0rgwtRskROAj919beygdoaZ7dcwvmpmxxF+j9bFjWpHqRjvBd5w99tbaJYXr0uSvuTR61JqZnun7ncF/h74S5NmHZbDkmyMnlM82U5e9wK/MbOVhHfHofEiblnCvlxqZmcDWwl9GR4t4C9gZr8jzJ7oaWY1wATCBSrc/R5gLmGmyErgU+A7cSJtXYK+fBP4gZltBTYBQ3O0qKgA/gV4LTV+DHA10Bfy7nVJ0pd8eV32B2aaWQnhjWmOuz+RrRymT+SKiBSRfBzeERGRnaSkLyJSRJT0RUSKiJK+iEgRUdIXESkiSvoiIkVESV9EpIgo6YuIFJH/BWI1/VWpUSuDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"l2_rate大小对于准确率的影响\")\n",
    "pyplot.plot(acc_array.keys(), acc_array.values(), 'b-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1435\n",
      "batch 1000 acc is 0.9735\n",
      "batch 2000 acc is 0.9754\n",
      "batch 3000 acc is 0.9816\n",
      "batch 4000 acc is 0.9814\n",
      "batch 5000 acc is 0.9845\n",
      "batch 6000 acc is 0.9842\n",
      "batch 7000 acc is 0.9837\n",
      "batch 8000 acc is 0.984\n",
      "batch 9000 acc is 0.9846\n",
      "0.9842\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1011\n",
      "batch 1000 acc is 0.9716\n",
      "batch 2000 acc is 0.9759\n",
      "batch 3000 acc is 0.9798\n",
      "batch 4000 acc is 0.9818\n",
      "batch 5000 acc is 0.9819\n",
      "batch 6000 acc is 0.9818\n",
      "batch 7000 acc is 0.9824\n",
      "batch 8000 acc is 0.9823\n",
      "batch 9000 acc is 0.9823\n",
      "0.9823\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1609\n",
      "batch 1000 acc is 0.969\n",
      "batch 2000 acc is 0.9741\n",
      "batch 3000 acc is 0.9804\n",
      "batch 4000 acc is 0.9795\n",
      "batch 5000 acc is 0.9815\n",
      "batch 6000 acc is 0.9813\n",
      "batch 7000 acc is 0.9824\n",
      "batch 8000 acc is 0.9816\n",
      "batch 9000 acc is 0.9819\n",
      "0.9823\n",
      "评分最好的l2_rate和acc是：0.0001,0.9842\n"
     ]
    }
   ],
   "source": [
    "acc_array = {}\n",
    "\n",
    "best_j = 0\n",
    "best_acc = 0\n",
    "l2_rates = [0.0001,0.00005,0.00001]\n",
    "for j in range(len(l2_rates)):\n",
    "    l2_rate = l2_rates[j]\n",
    "    _layer_nodes = [800,160] #[600,500,400,10]\n",
    "    accuraccy = train_withLearningRate_L2(layer_nodes = _layer_nodes,use_l2 = True, l2_rate = l2_rate)\n",
    "    acc_array[j] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_j = j\n",
    "    \n",
    "print(\"评分最好的l2_rate和acc是：\"+ str(l2_rates[best_j]) + \",\" + str(best_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#包装函数方便调优\n",
    "\n",
    "def train_withLearningRate_L2_with_diff_function(layer_nodes = [500,200], use_l2 = False, l2_rate = 0.0001, sti_func =tf.nn.relu):\n",
    "    # Create the model\n",
    "    x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "\n",
    "    \n",
    "    y = x\n",
    "    current_node = 784\n",
    "    WW = []\n",
    "    for i in range(len(layer_nodes)):\n",
    "        node_count = layer_nodes[i]\n",
    "        print(str(current_node) + \"_\" + str(node_count))\n",
    "        W = tf.Variable(tf.truncated_normal([current_node, node_count],stddev = 0.1))\n",
    "        b = tf.Variable(tf.constant(0.1,shape = [node_count]))\n",
    "        y = sti_func(tf.matmul(y, W) + b)\n",
    "        current_node = node_count\n",
    "        WW.append(W)\n",
    "    W = tf.Variable(tf.truncated_normal([current_node, 10],stddev = 0.1))\n",
    "    WW.append(W)\n",
    "    b = tf.Variable(tf.constant(0.1,shape = [10]))\n",
    "    y = tf.matmul(y, W) + b\n",
    "    \n",
    "   \n",
    "    y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "    # The raw formulation of cross-entropy,\n",
    "    #\n",
    "    #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "    #                                 reduction_indices=[1]))\n",
    "    #\n",
    "    # can be numerically unstable.\n",
    "    #\n",
    "    # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "    # outputs of 'y', and then average across the batch.\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    "    )\n",
    "    loss = cross_entropy\n",
    "    \n",
    "    if use_l2:\n",
    "        #计算l2正则化损失函数\n",
    "        regularizer = tf.contrib.layers.l2_regularizer(l2_rate)\n",
    "        #计算模型的正则化损失，只计算权重，不使用b\n",
    "        regularization = 0\n",
    "        for w in WW:\n",
    "            regularization =regularization + regularizer(w)\n",
    "        loss = loss + regularization\n",
    "    \n",
    "    batchSize = 100\n",
    "    trainingStep = 10000\n",
    "    Learning_rate_base = 0.5\n",
    "    Learning_rate_decay = 0.99\n",
    "    global_step = tf.Variable(0,trainable = False)\n",
    "    \n",
    "    Learning_rate = tf.train.exponential_decay(\n",
    "        Learning_rate_base,\n",
    "        global_step,\n",
    "        mnist.train.num_examples / batchSize,\n",
    "        Learning_rate_decay)\n",
    "    \n",
    "    train_step = tf.train.GradientDescentOptimizer(Learning_rate).minimize(loss,global_step = global_step)\n",
    "\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    # Train\n",
    "    for _ in range(trainingStep):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batchSize)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        \n",
    "        if _ %1000 == 0:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            acc = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "            print(\"batch \" + str(_) + \" acc is \" + str(acc))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    ret = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "    print(ret)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1331\n",
      "batch 1000 acc is 0.9703\n",
      "batch 2000 acc is 0.9764\n",
      "batch 3000 acc is 0.9767\n",
      "batch 4000 acc is 0.9813\n",
      "batch 5000 acc is 0.9818\n",
      "batch 6000 acc is 0.9836\n",
      "batch 7000 acc is 0.9835\n",
      "batch 8000 acc is 0.9838\n",
      "batch 9000 acc is 0.9839\n",
      "0.9838\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1028\n",
      "batch 1000 acc is 0.9223\n",
      "batch 2000 acc is 0.9442\n",
      "batch 3000 acc is 0.9491\n",
      "batch 4000 acc is 0.9551\n",
      "batch 5000 acc is 0.96\n",
      "batch 6000 acc is 0.9641\n",
      "batch 7000 acc is 0.9654\n",
      "batch 8000 acc is 0.9669\n",
      "batch 9000 acc is 0.9679\n",
      "0.9708\n",
      "784_800\n",
      "800_160\n",
      "batch 0 acc is 0.1443\n",
      "batch 1000 acc is 0.9648\n",
      "batch 2000 acc is 0.9721\n",
      "batch 3000 acc is 0.976\n",
      "batch 4000 acc is 0.9784\n",
      "batch 5000 acc is 0.9785\n",
      "batch 6000 acc is 0.9789\n",
      "batch 7000 acc is 0.98\n",
      "batch 8000 acc is 0.98\n",
      "batch 9000 acc is 0.9791\n",
      "0.9809\n",
      "评分最好的l2_rate和acc是：<function relu at 0x0000000007D75598>,0.9838\n"
     ]
    }
   ],
   "source": [
    "acc_array = {}\n",
    "\n",
    "best_j = 0\n",
    "best_acc = 0\n",
    "sti_funcs = [tf.nn.relu,tf.nn.sigmoid,tf.nn.tanh]\n",
    "for j in range(len(l2_rates)):\n",
    "    sti_func = sti_funcs[j]\n",
    "    _layer_nodes = [800,160] #[600,500,400,10]\n",
    "    accuraccy = train_withLearningRate_L2_with_diff_function(layer_nodes = _layer_nodes,use_l2 = True, l2_rate = 0.0001, sti_func = sti_func)\n",
    "    acc_array[j] = accuraccy\n",
    "    if accuraccy > best_acc:\n",
    "        best_acc = accuraccy\n",
    "        best_j = j\n",
    "    \n",
    "print(\"评分最好的l2_rate和acc是：\"+ str(sti_funcs[best_j]) + \",\" + str(best_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "relu的结果更好一些"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的 ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits 的 logits 参数是**未经激活的 wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个 batch。\n",
    "然后我们运行 3k 个 step (5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到 92% 左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至 98% 以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个 step 就对 loss、accuracy 等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
